Friday, September 23, 2016

Youtube 360 비디오 제작 방법


Youtube 360 Video 키워드로 찾아 들어오시는 분들이 많은 것 같아서 좀 더 상세하게 방법을 적어 볼까 합니다.

최적화된 방법은 아니고, 간단하고 직관적으로 만들어본 것이기 때문에 좀더 고급 용도로 사용하시기 위해선 유료 에셋(예를 들어 이것, 저도 써보지는 못했습니다.)을 쓰시는 것이 좋습니다.

개발 동기는 몰입형 가상현실 시설을 위해서 파노라마 렌더링 셰이더를 만든 것이고, 원래 여기에서는 대략 가로 180도, 세로 50도 정도의 시야각만이 필요해서 3대의 카메라만을 사용하는 셰이더를 만들었습니다.

여기서 6개로 확장하는 것은 크게 어렵지 않습니다. 아래는 제가 만든 셰이더 입니다.

Shader "Unlit/MyPanoEntireOrtho"
{
 Properties
 {
  _Center("Center", 2D) = "white" {}
 _Left("Left", 2D) = "white" {}
 _Right("Right", 2D) = "white" {}
 _Top("Top", 2D) = "white" {}
 _Bottom("Bottom", 2D) = "white" {}
 _Back("Back", 2D) = "white" {}
 
 }
  SubShader
 {
  Tags{ "RenderType" = "Opaque" }
  LOD 100

  Pass
 {
  CGPROGRAM
#pragma vertex vert
#pragma fragment frag
  // make fog work
#pragma multi_compile_fog
#pragma target 3.0

#include "UnityCG.cginc"

 struct appdata
 {
  float4 vertex : POSITION;
  float2 uv : TEXCOORD0;
 };

 struct v2f
 {
  float2 uv : TEXCOORD0;
  UNITY_FOG_COORDS(1)
   float4 vertex : SV_POSITION;
 };

#define PI 3.141592653589793
#define HALFPI 1.57079632679

 sampler2D _Center;
 sampler2D _Left;
 sampler2D _Right;
 sampler2D _Top;
 sampler2D _Bottom;
 sampler2D _Back;
 float _VFOV;
 float4 _MainTex_ST;

 v2f vert(appdata v)
 {
  v2f o;
  o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
  o.uv = v.uv;
  return o;
 }

 
 fixed4 frag(v2f i) : SV_Target
 {
  //half4 texcol = tex2D(_MainTex,i.uv);

  //Image texture coord --> lat, lon angle
  float latDeg = -180 + i.uv.x * 360;
  float lonDeg = -90 + i.uv.y * 180;
  float latRad = latDeg*PI / 180;
  float lonRad = lonDeg*PI / 180;

 //lat, lon --> unit sphere xyz
 //View dir = (1,0,0)
 float3 vec = float3(1, 0, 0);
 float3x3 rotLat = float3x3(cos(latRad), -sin(latRad), 0, sin(latRad), cos(latRad), 0, 0, 0, 1);
 float3x3 rotLon = float3x3(cos(lonRad), 0, sin(lonRad), 0, 1, 0, -sin(lonRad), 0, cos(lonRad));
 float3 rot = mul(rotLon, vec);
 float3 SphereXYZ = mul(rotLat, rot);

 //If ray hits image(center,left,right) plane, assign that value
 //Center 평면과 접점(X = 1)
 //if (SphereXYZ.x < 0) return half4(0, 1, 0, 1);

 float3 fVec = SphereXYZ / SphereXYZ.x;
 float3 lVec = SphereXYZ / -SphereXYZ.y;
 float3 rVec = SphereXYZ / SphereXYZ.y;
 float3 bVec = SphereXYZ / -SphereXYZ.z;
 float3 tVec = SphereXYZ / SphereXYZ.z;
 float3 backVec = SphereXYZ / -SphereXYZ.x;

 if (abs(fVec.y) <= 1 && abs(fVec.z) <= 1 && SphereXYZ.x >= 0)
 {
  float pU = fVec.y / 2 + 0.5;
  float pV = -fVec.z / 2 + 0.5;
  return tex2D(_Center, float2(pU, pV));
 }

 if (abs(lVec.x) <= 1 && abs(lVec.z) <= 1 && SphereXYZ.y < 0)
 {
  float pU = lVec.x / 2 + 0.5;
  float pV = -lVec.z / 2 + 0.5;
  return tex2D(_Left, float2(pU, pV));

 }

 if (abs(rVec.x) <= 1 && abs(rVec.z) <= 1 && SphereXYZ.y >= 0)
 {
  float pU = -rVec.x / 2 + 0.5;
  float pV = -rVec.z / 2 + 0.5;
  return tex2D(_Right, float2(pU, pV));

 }

 if (abs(bVec.x) <= 1 && abs(bVec.y) <= 1 && SphereXYZ.z < 0)
 {
  float pU = bVec.y / 2 + 0.5;
  float pV = -bVec.x / 2 + 0.5;
  return tex2D(_Top, float2(pU, pV));

 }

 if (abs(tVec.x) <= 1 && abs(tVec.y) <= 1 && SphereXYZ.z >= 0)
 {
  float pU = tVec.y / 2 + 0.5;
  float pV = tVec.x / 2 + 0.5;
  return tex2D(_Bottom, float2(pU, pV));

 }

 if (abs(backVec.y) <= 1 && abs(backVec.z) <= 1 && SphereXYZ.x < 0)
 {
  float pU = -backVec.y / 2 + 0.5;
  float pV = -backVec.z / 2 + 0.5;
  return tex2D(_Back, float2(pU, pV));

 }

 return half4(0, 1, 0, 1);
 
 }
  ENDCG
 }
 }
}

쓰잘데기 없이 깁니다만 간단히 요약하면 결과 텍스처의 각 uv좌표를 가지고 위도와 경도를 계산하고, 그 위도 경도에 해당하는 픽셀값이 Center/Left/Right/Top/Bottom/Back 중 어떤 이미지에 담겨있는지 판별하고, 해당 이미지의 uv좌표로 변환해서 픽셀값을 샘플링 하는 것입니다.

이제 이 셰이더를 유니티에서 활용하려면, 셰이더를 asset에 추가하고, 새로운 material("Pano"로 이름짓겠습니다)을 만들어서 Unlit/MyPanoEntireOrtho 셰이더를 할당해줍니다.

그리고 6개의 Render Texture (Center/Left/Right/Top/Bottom/Back)를 만들어줍니다.

유니티의 Scene에서는 1개의 GameObject("PanoCam"으로 이름짓겠습니다.)와 6개의 카메라를 만들어주고, Field of view는 90으로 잡아줍시다. 그리고 6개의 카메라를 PanoCam의 자식으로 달아줍니다. 각 카메라의 Euler Angle은 아래와 같습니다.

Front : [0,0,0]
Left : [0,270,0]
Right : [0,90,0]
Top : [270,0,0]
Bottom : [90,0,0]
Back : [0,180,0]

그러면 아래 그림과 같이 6개의 카메라가 전 방향을 커버하게 됩니다. 이제 각 카메라의 Target Texture에 아까 만들었던 6개의 Render Texture를 하나씩 할당해 줍니다.


그리고 나서 Render Texture들을 끌어다 "Pano" Material의 텍스처들에 할당해주면 아래와 같은 그림을 보실 수 있을겁니다.


이렇게 하고나서 "PanoCam" GameObject를 움직여보면 Material이 바뀌는 것을 보실 수 있습니다.

이제 마지막으로 이 Material을 최종적으로 화면에 뿌려주어야 합니다. Graphics.Blit을 사용하셔도 되는데, 저는 보통 간단하게 물리적으로 해결합니다.

추가로 카메라를 하나 더 만들고("MainCam"이라고 명명), Plane을 하나 만들어줍니다. MainCam의 Projection은 Orthographic으로 하고, size는 5로 해줍니다. Plane은 "MainCam"의 자식으로 놓고, 위치는 [0,0,1], Euler Angle은 [90,180,0], Scale은 [1.777777,1,1]로 해줍니다. 16:9 화면비 기준입니다.

그리고 나서 Plane의 Material로 "Pano" Material을 해 주시면 이제 게임 프리뷰에서 16:9 화면에 꽉 차게 파노라마 이미지가 나올 것입니다.


이제는 빌드 후 실행을 하기만 하면 스크린 캡처를 통해서 360 영상을 만들 수 있습니다.

여유가 좀 되신다면, 원본 글에도 적었던 AVPro Movie Capture 에셋을 사용하는 것이 가장 좋습니다. 특히 화면 해상도를 올리고 싶다면 거의 필수입니다. 고해상도 영상 녹화가 가능하도록 하는 모듈을 직접 만들어 보고 싶지만 잘 모르는 분야라서 쉽지 않을 것 같네요.

아니면 지금 보여드리는 예제 영상에서 제가 한것처럼, Fraps, oCam, Camtasia 등 스크린 캡쳐 프로그램을 사용하셔도 됩니다. 아래는 메타데이터 없이 유튜브에 업로드된 동영상입니다.

이제, 360 영상을 유튜브에서 볼 수 있도록 하기 위해 메타데이터를 추가해 주어야 합니다. 여기의 2번에 보시면 맥과 윈도우용 프로그램을 다운로드 할 수 있습니다.

다운로드 후에 프로그램을 실행해 보시면 바로 사용법을 아실 수 있습니다. Open으로 캡처한 영상을 불러오고, Spherical에 체크 해준 후에 Save as하여 저장하시면 됩니다. 아래는 성공적으로 메타데이터가 추가된 모습입니다.


이제 유튜브에 영상을 게시하시면 아래와 같이 360도 영상을 볼 수 있습니다. 이상하게 갑자기 제 컴퓨터에선 크롬을 통해서는 잘 안보이네요. 익스플로러에서는 잘 보입니다.




Tuesday, July 26, 2016

CMU Motion Capture Database Viewer ver. 1.0

Today I opened CMU Motion Capture Database Viewer webpage. You can connect by clicking the tab located above from this blog or the direct link below.


You can select a subject and corresponding data by selection list, then the animation will be visualized in X3DOM canvas immediately. 

If you have any feedback, please let me know.


Tuesday, July 19, 2016

asf/amc to X3D H-Anim conversion

Recently, we're working on human gait analysis-related research. While studying related literature, we found CMU Graphics Lab Motion Capture Database, which has huge amount of motion capture data using Vicon motion capture system. Database includes video taken while capturing motion (mpg), sensor raw data (c3d) ,reconstructed skeleton from sensor data (asf/amc) and animated video of skeleton (avi). There are tvd files you can download but I couldn't get much information about the file format.

The motivation of this work is that it is difficult to find the motion capture data that I wanted. For instance, I wanted to find the data of walking along the circular path over a minute. All you can do is search the database with keyword "walking" and should download each mpg or avi video and check it is longer than a minute and if a person walked along a circular path. It is very exhaustive work and I found it is faster to download entire avi clips in database and see the videos one by one (because downloading each avi in searched result takes too much time).

So my idea is to make a webpage using X3DOM, shows skeleton animation in interactive 3D without manually download files. It is possible because H-Anim specification is ready for the standardized character animation description and thankfully, X3DOM developers implemented the specification.

Therefore I implemented a converter from asf/amc to X3D H-Anim. The converted x3d file can be visualized in X3DOM framework. To reduce the work needed I use the exist MATLAB parser for asf/amc. If you are Korean, this page helps you to understand how asf/amc format describes skeleton data.

The important and difficult part of converting asf/amc to H-Anim specification is the existence of "axis" information in asf file format, and inverted multiplication order or rotation matrix between those two. In asf, local axis of a idle(neutral?) bone may not aligned with global coordinate. However in H-Anim, bone (joint in H-Anim specification) is aligned with global coordinate. So you have to convert animation data stored in amc by changing local rotation to global rotation. And asf/amc descripbes left-to-right multiplication of rotation matrix, while rotation matrix of H-Anim should be right-to-left.

Below is an example of X3DOM scene, converted from asf/amc. (I think Google Blogger now blocks external js include...?)

Link


Now I'm converting the entire database (which takes so much time because MATLAB is not good for this kind of work...) and I planned to make a separate page to visualize CMU database using X3DOM. I hope to finish and open it in near future.


Monday, March 21, 2016

Youtube 360 Video


  • 제작 방법 상세 설명 추가 (여기)
  • https://support.google.com/youtube/answer/6178631?hl=ko
  • You can change view angle while watching Youtube video if video was produced by equirectangular projected 360 image.

  • 1920x1080p 




Friday, March 18, 2016

Panoramic Rendering for CAVE facility

Overview


  • We have new CAVE in our lab consisted of three affordable home projectors-have low ANSI Lumens with full HD resolution, and screen that covers 180 degree FOV in horizon with spherical surface.


Screen configuration (right picture is top view)
  • We decided not to spend the time or budget for synchronization of clustered rendering. Instead, we installed Quadro to our rendering node.
  • Actual dimensions of a screen we measured and installed picture is shown below. You may notice the black cloths are attached on the screen. Because our projector have short throw distance, installation guys decided not to use some part of the screen. In diagram, that part was drawn like those covers same arc length in top part and bottom part, but actually it is not. I will discuss about it in later post. In this post, I ignored the asymmetry.
  • Actual radius is about 2.5m and vertical FOC is about 48.91 degree.
Picture of the installed screen and projectors


Warping and edge blending

  • Warping and edge blending is necessary for multi-projector system. You may want to get more information in http://paulbourke.net/
  • The installation company did this job for us using their calibration software and hardware. We don't have much things to do up to this point.
  • The picture before/after warping and edge blending is shown below.
(top) Before edge blending and warping (bottom) After edge blending and warping
  • After warping and edge blending is done, the image generated by our rendering node properly mapped to screen. The relation between four corners of rendered image and of screen is like below diagram, if we assume 100% accurate warping.


Panoramic rendering for 180 degree horizontal FOV


  • Unfortunately, this is not the end of the process if you want deliver the final image to user properly.
  • The first problem is that the image generated by single perspective camera cannot cover 180 degree FOV. Let's say we generate image has 5760x1080 pixels while did not changed default FOV set in Unity3D's camera (60 degree vertical FOV). Then by defined aspect ratio, rendered image contains only pixel information of about 144 degree horizontal FOV.
  • You can imagine how the image on our screen can be distorted by the horizontally "stretched" mapping, by diagram below.

Solution : Panoramic rendering

  • To solve this problem, we need panoramic rendering using several cameras.
  • Simply, we need the final image that has pixel information of [-90 deg, -24.46 deg] from camera on [0,0] uv coordinate and [90 deg, 24.46 deg] on [1,1] uv coordinate.
  • Usually you may want to use cubemap and map it to single image using equirectangular projection. But I decided to implement custom shader for study and I guess there should be performance problem by rendering 6 images instead of 3 that is all what we need to cover 180 and 48.91 deg. Also we can optimize the resolution by not producing power of two resolution texture that Unity is forcing for cubemap.
  • So I have three camera in my scene(front/left/right) each renders scene of 90 by 90 degree region. From those three rendered texture, pixel values of final 5760x1080 image is calculated by shader with process below.
    • Calculate corresponding latitude and longitude angle from UV coordinate of final image
    • Convert latitude and longitude angles to unit direction vector
    • Extend vector(ray) to front/left/right image plane
    • If ray hits, calculate local UV coordinates for hit position in front/left/right image plane
    • Return tex2D(front/left/right image , local UV coordinate), which is pixel value

Result video clip

  • Left quarter shows 3D scene and camera(red cube) and right image shows rendered final image

Adjusting projection point to actual eye position

  • Above final image seems okay, but we can also consider the actual user's eye position in the facility.
  • As you can see in the diagram below, the altitude of user's eye position is not located at the center of screen sphere.

Solution : Changing projection origin

  • We can calculate asymmetric upper and lower viewing angle(18.74 deg, 38.36 deg) from screen radius(2.5m), screen sphere to ground distance(1.25m), half of vertical FOV from sphere origin(24.46 deg) and actual eye position to ground distance(1.70m).
  • The process of panoramic rendering is similar with previous one. Changed part is the map between UV coords and lat, lon angle and changed origin to the actual eye position.
Diagram of actual eye position, ground plane and projection screen
Changed uv-lat,lon mapping

  •  Another change is that now we need bottom camera since now bottom part is shown a little due to the extension of lower viewing angle from 22.46 deg to 38.36 deg. Adding pixel information from bottom image to panoramic image is similar with others(front/left/right).
Final image from adjusted projection point and front/left/right image. Green pixels shows no pixel information exists
Filled pixel information from adding bottom image

Result video clip



  • You can see the asymmetry of viewing angle between upper and lower in vertical.

Observing result with application

  • We have excavator simulator for validation of safety control algorithm. Below picture shows the application rendered on screen 1) without any processing, 2) with panoramic rendering and 3) with panoramic rendering and eye position adjusting.
  • I take pictures using panorama capturing function on my smart phone. Three results are very different if you stand in front of the projection screen but it may seems not that dramatic in took pictures below. But I cannot find other better way to show the result.
Scene without any processing. 144 HFOV, 60 VFOV

Scene with panoramic rendering. 180 HFOV, 48.91 VFOV
Scene with panoramic rendering and eye position adjustment. 180 HFOV, 57.1 VFOV 
  • You may notice the effect of eye position adjustment by seeing the building in the 3D scene. Picture was taken right in front of my eye position.

The building in the left side in panoramic rendering scene
The building in the left side in panoramic rendering scene with eye position adjustment