Record a 60fps movie (with sound) using Uzem

The Uzebox now have a fully functional emulator! Download and discuss it here.
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

uze6666 wrote: :shock: You totally convinced me! The quality is amazing and the whole process is simple...I love it, you the man! :mrgreen:

I'm trying it tonight...If all goes well, I'll report soon.

ps: Your game is looking great!
Hehe, thanks! :-D

I think the sounds are still a tiny bit off, I suspect that the mp3 encoder can't handle the 15700 sample rate, and that it is just treating it as if it were 16000. I'll experiment with upsampling the sound as well to see if that fixes it.
User avatar
uze6666
Site Admin
Posts: 4801
Joined: Tue Aug 12, 2008 9:13 pm
Location: Montreal, Canada
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by uze6666 »

Now I'm *really* impressed. Downloaded ffmpeg.exe and put it in the same folder as uzem, patched your code and...it worked the first time! Perhaps it's my old CPU or my HD but it seriously slows down my machine. But when played back it's flawless and full speed, so pretty awesome! :mrgreen:

Can you post the full patch with your 640x480 fix? I must forget something because it crashes.
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

uze6666 wrote:Now I'm *really* impressed. Downloaded ffmpeg.exe and put it in the same folder as uzem, patched your code and...it worked the first time! Perhaps it's my old CPU or my HD but it seriously slows down my machine. But when played back it's flawless and full speed, so pretty awesome! :mrgreen:

Can you post the full patch with your 640x480 fix? I must forget something because it crashes.
Sweet! I wonder if the newest ffmpeg decides to use all the cores that it can for encoding, and that's why it's slowing down your machine?

Here is my latest patch. I also added a bit of code that calls shutdown after it finishes playing back an input capture file, so I can run consistent tests. I also added a call to SDL_Quit() so in case the call to ffmpeg at the end takes more "happens instantly" you'll be able to hit Ctrl-C to cancel it.

Code: Select all

diff --git a/tools/uzem/avr8.cpp b/tools/uzem/avr8.cpp
index 77698bb..7df2526 100644
--- a/tools/uzem/avr8.cpp
+++ b/tools/uzem/avr8.cpp
@@ -236,6 +236,9 @@ void avr8::spi_calculateClock(){
     SPI_DEBUG("SPI divider set to : %d (%d cycles per byte)\n",spiClockDivider,spiCycleWait);
 }
 
+FILE* avconv_video = NULL;
+FILE* avconv_audio = NULL;
+
 void avr8::write_io(u8 addr,u8 value)
 {
 	if (addr == ports::PORTC)
@@ -282,6 +285,9 @@ void avr8::write_io(u8 addr,u8 value)
             	SDL_framerateDelay(&fpsmanager);
 
                 SDL_Event event;
+		if (avconv_video == NULL) avconv_video = popen("avconv -y -f rawvideo -s 640x480 -pix_fmt bgra -r 60 -i - -vf scale=-1:720 -an -b:v 1000k test.mp4", "w");
+		if (avconv_video) fwrite(screen->pixels, 640*480*4, 1, avconv_video);
+
                 while (singleStep? SDL_WaitEvent(&event) : SDL_PollEvent(&event))
                 {
 					switch (event.type) {
@@ -316,6 +322,9 @@ void avr8::write_io(u8 addr,u8 value)
                 	buttons[0]=captureData[capturePtr]+(captureData[capturePtr+1]<<8);
                 	capturePtr+=2;
                 	captureSize-=2;
+                }else if(captureSize==0){
+			printf("Playback reached end of capture file.\n");
+			shutdown(0);
                 }
 
 
@@ -459,6 +468,8 @@ void avr8::write_io(u8 addr,u8 value)
 			SDL_LockAudio();
 			audioRing.push(value);
 			SDL_UnlockAudio();
+			if (avconv_audio == NULL) avconv_audio = popen("avconv -y -f u8 -ar 15700 -ac 1 -i - -codec:a mp3 -ar 44.1k test.mp3", "w");
+			if (avconv_audio) fwrite(&value, 1, 1, avconv_audio);
 		}
 	}
 
@@ -1377,7 +1388,7 @@ bool avr8::init_gui()
 	init_joysticks();
 
 	if (fullscreen)
-		screen = SDL_SetVideoMode(800,600,32,sdl_flags | SDL_FULLSCREEN);
+		screen = SDL_SetVideoMode(640,480,32,sdl_flags | SDL_FULLSCREEN);
 	else
 		screen = SDL_SetVideoMode(630,448,32,sdl_flags);
 	if (!screen)
@@ -1386,7 +1397,7 @@ bool avr8::init_gui()
 		return false;
 	}
 	else if (fullscreen)	// Center in fullscreen
-		inset = ((600-448)/2) * screen->pitch + 4 * ((800-630)/2);
+		inset = ((480-448)/2) * screen->pitch + 4 * ((640-630)/2);
 
 	if (SDL_MUSTLOCK(screen) && SDL_LockSurface(screen) < 0)
 		return false;
@@ -2500,6 +2511,17 @@ void avr8::shutdown(int errcode){
     	fclose(captureFile);
     }
 
+    if (avconv_video) pclose(avconv_video);
+    if (avconv_audio) pclose(avconv_audio);
+    FILE* avconv_mux = popen("avconv -y -i test.mp4 -i test.mp3 -vcodec copy -acodec copy -f mp4 uzem.mp4", "r");
+
+    if (avconv_mux) {
+	SDL_Quit();
+	pclose(avconv_mux);
+	unlink("test.mp4");
+	unlink("test.mp3");
+    }
+
 #if GUI
 	if (joystickFile) {
 		FILE* f = fopen(joystickFile,"wb");
Edit: Updated patch so the audio gets upsampled, and now it sounds correct.
Last edited by Artcfox on Thu Sep 24, 2015 3:11 am, edited 1 time in total.
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

Yup, my guess about the audio was correct. Apparently MPEG-2 Layer III audio can only support a handful of sample rates.

You can correct the sound by changing the line that sets up the audio to this:

Code: Select all

			if (avconv_audio == NULL) avconv_audio = popen("avconv -y -f u8 -ar 15700 -ac 1 -i - -codec:a mp3 -ar 44.1k test.mp3", "w");
(It will do the mp3 sample rate conversion and encoding on the fly in parallel in another process.)

and then in the shutdown function, change it to:

Code: Select all

    FILE* avconv_mux = popen("avconv -y -i test.mp4 -i test.mp3 -vcodec copy -acodec copy -f mp4 uzem.mp4", "r");
(Now no actual encoding happens when you're trying to quit the emulator, it's just going to instantly combine the audio and video streams into a single file.)

and change the unlink call to:

Code: Select all

	unlink("test.mp3");
I updated the patch file in my previous post with these changes, and I also fixed the example video that I uploaded to have the correct sound.
User avatar
uze6666
Site Admin
Posts: 4801
Joined: Tue Aug 12, 2008 9:13 pm
Location: Montreal, Canada
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by uze6666 »

Great! I'll check those sound settings.

Btw, the movie capture can work no problems in window mode too (which I prefer). I'll just make the window 640x480 instead and always set the "inset" variable.

I also have tested different scaling algorithms and it seems nearest neighbor yield the best results (more sharp).

Code: Select all

if (avconv_video == NULL) avconv_video = popen("ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgra -r 60 -i - -vf scale=-1:720 -sws_flags neighbor -an -b:v 1000k test.mp4", "w");
Ghosty Ghost 60pfs :mrgreen:
https://www.youtube.com/watch?v=WEOmuqH76sk

Edit:With you latest audio fix, it actually sound much clearer too! The bit rate seems much higher.
https://www.youtube.com/watch?v=3v8gUmlFM2M
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

uze6666 wrote:Great! I'll check those sound settings.

Btw, the movie capture can work no problems in window mode too (which I prefer). I'll just make the window 640x480 instead and always set the "inset" variable.

I also have tested different scaling algorithms and it seems nearest neighbor yield the best results (more sharp).

Code: Select all

if (avconv_video == NULL) avconv_video = popen("ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgra -r 60 -i - -vf scale=-1:720 -sws_flags neighbor -an -b:v 1000k test.mp4", "w");
Ghosty Ghost 60pfs :mrgreen:
https://www.youtube.com/watch?v=WEOmuqH76sk

Edit:With you latest audio fix, it actually sound much clearer too! The bit rate seems much higher.
https://www.youtube.com/watch?v=3v8gUmlFM2M
Cool. I originally tried to support both, but the encoder was not happy with the 630x448 window size, so that's why I started focusing on the fullscreen mode (and I wasn't sure how you'd feel about me changing the size of the window mode). Window mode just makes everything run in slow motion for me (on my laptop), so I don't use it much. It would definitely be nicer to multitask while Uzem is recording an input capture, instead of taking over the entire screen. Now I really want to try this on my desktop machine, because I'd bet that I'd be able to record while I'm playing the game at full speed in fullscreen 640x480 mode, since it has so many cores.

What are your thoughts about the code I added to call shutdown when the input capture is done being played back? I was trying out some crazy filters, and I didn't want to have to babysit the computer in order to stop the recording after playing back a very long input capture.

Good catch on the nearest neighbor scaling! I thought that it was already using that, but now that I've compared them against each other I guess it wasn't. Nearest neighbor also makes the file size of the encoded video smaller.

Ghosty Ghost looks great!

I wonder if we should just set sane defaults, but allow each encoding command to be completely overridden using environment variables, since there are so many things that a user might want to change. For instance, I'm really curious what the xbr=4 filter would look like, but my version of avconv doesn't support any of the pixel-art specific upscalers.
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

Uze, could you try it with this audio encoding line instead?

Code: Select all

if (avconv_audio == NULL) avconv_audio = popen("avconv -y -f u8 -ar 15700 -ac 1 -i - -acodec libmp3lame -ar 44.1k test.mp3", "w");
It turns out that on Debian, there is a compatibility script called ffmpeg that was installed and I think it uses the original style ffmpeg arguments. I'm trying to find a single set of arguments that works for all instances, and then we can just pretend that everyone has ffmpeg installed and forget that avconv even exists.

Edit: It turns out that using the compatibility script works, but it make everything super slow, presumably because it's passing the input stream through a script. Really I just want you to test that -acodec libmp3lame part works for you too.
User avatar
uze6666
Site Admin
Posts: 4801
Joined: Tue Aug 12, 2008 9:13 pm
Location: Montreal, Canada
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by uze6666 »

What are your thoughts about the code I added to call shutdown when the input capture is done being played back? I was trying out some crazy filters, and I didn't want to have to babysit the computer in order to stop the recording after playing back a very long input capture.
We can make it the default yes. After all, the point is to replay the game like a movie so when its done, its done.
I wonder if we should just set sane defaults, but allow each encoding command to be completely overridden using environment variables, since there are so many things that a user might want to change. For instance, I'm really curious what the xbr=4 filter would look like, but my version of avconv doesn't support any of the pixel-art specific upscalers.
Perhaps some config file would be a better choice for this and future stuff too. I have tried xbr, hqx,super2xsai but they don't seem to work as expected probably because the image fed is already pre-scaled by uzem. In any cases it slows down even more my machine. Fortunately there's the capture function!
if (avconv_audio == NULL) avconv_audio = popen("avconv -y -f u8 -ar 15700 -ac 1 -i - -acodec libmp3lame -ar 44.1k test.mp3", "w");
Yes, this works fine with ffmpeg.
User avatar
uze6666
Site Admin
Posts: 4801
Joined: Tue Aug 12, 2008 9:13 pm
Location: Montreal, Canada
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by uze6666 »

I have pushed in the uzem140 branch the code to record movies.

In this version, simply add the -r switch to record a movie. The base filename will be the rom name and the format 720p(60fps) to be easily uploadable to Youtube. Download the ffmpeg static executable and insure its in your path. That's it. https://www.ffmpeg.org/download.html. As suggested by Artfox, depending on the os and your machine's power, it could slow down, but if it does, use the capture feature to play full speed that record the movie while playing back the capture. The capture mode also automatically exits when the capture is done.

Oh, and thanks Artfox for all those goodies!! :mrgreen:
User avatar
Artcfox
Posts: 1382
Joined: Thu Jun 04, 2015 5:35 pm
Contact:

Re: Record a 60fps movie (with sound) using Uzem

Post by Artcfox »

uze6666 wrote:I have pushed in the uzem140 branch the code to record movies.

In this version, simply add the -r switch to record a movie. The base filename will be the rom name and the format 720p(60fps) to be easily uploadable to Youtube. Download the ffmpeg static executable and insure its in your path. That's it. https://www.ffmpeg.org/download.html. As suggested by Artfox, depending on the os and your machine's power, it could slow down, but if it does, use the capture feature to play full speed that record the movie while playing back the capture. The capture mode also automatically exits when the capture is done.

Oh, and thanks Artfox for all those goodies!! :mrgreen:
I guess I should have checked this thread first! I just finished making my own patch that makes recording optional. Take a look at my patch, because you'll probably want to change some things in the version you just pushed to the uzem140 branch. I made ffmpeg_video and ffmpeg_audio proper class variables, and moved their initialization into uzem.cpp. That ensures that both streams get created if you're asking to record things, which matters if the user specifically requests that sounds are disabled. (Creating both streams up front makes sure that the combining step works regardless of whether sound is disabled or not, without having to specifically test for that.) Also, that means I didn't need to create a new flag to say if recording is enabled, because the stream pointers will be non-zero, and that's what I test for when the emulator is shutting down.

Also, it looks like your patch got rid of the frame rate manager, without adding it back in. Wouldn't that make it so the emulator always renders as fast as it can? I would say to keep that exactly how it was, because for my next modification to Uzem I'm planning on assigning hotkeys that will call

Code: Select all

SDL_setFramerate(&fpsmanager, desiredFrameRate);
after incrementing or decrementing desiredFrameRate. That very simply adds the speed control that I had requested in a previous thread, and it even lets you hear the sound effects in slow motion. ;) Usually I want to watch the video recordings at actual speed, but in case I want to speed it up (or slow it down to see something), I can just hit a hotkey to change the framerate like that, while the recording stays perfectly locked to 60fps.

One other thing that my patch has is a bugfix for the 60fps switchover. (In SDL_framerate.h, FPS_DEFAULT was still set to 30, rather than 60.)

Here is the patch that I had prepared:

Code: Select all

diff --git a/tools/uzem/SDL_framerate.h b/tools/uzem/SDL_framerate.h
index 521797a..63e3245 100644
--- a/tools/uzem/SDL_framerate.h
+++ b/tools/uzem/SDL_framerate.h
@@ -54,7 +54,7 @@ extern "C" {
 	/*!
 	\brief Default rate of framerate controller in Hz (1/s).
 	*/
-#define FPS_DEFAULT		30
+#define FPS_DEFAULT		60
 
 	/*! 
 	\brief Structure holding the state and timing information of the framerate controller. 
diff --git a/tools/uzem/avr8.cpp b/tools/uzem/avr8.cpp
index 77698bb..112d04e 100644
--- a/tools/uzem/avr8.cpp
+++ b/tools/uzem/avr8.cpp
@@ -282,6 +282,9 @@ void avr8::write_io(u8 addr,u8 value)
             	SDL_framerateDelay(&fpsmanager);
 
                 SDL_Event event;
+
+		if (ffmpeg_video) fwrite(screen->pixels, 640*480*4, 1, ffmpeg_video);
+
                 while (singleStep? SDL_WaitEvent(&event) : SDL_PollEvent(&event))
                 {
 					switch (event.type) {
@@ -316,6 +319,9 @@ void avr8::write_io(u8 addr,u8 value)
                 	buttons[0]=captureData[capturePtr]+(captureData[capturePtr+1]<<8);
                 	capturePtr+=2;
                 	captureSize-=2;
+                }else if(captureSize==0){
+			printf("Playback reached end of capture file.\n");
+			shutdown(0);
                 }
 
 
@@ -459,6 +465,7 @@ void avr8::write_io(u8 addr,u8 value)
 			SDL_LockAudio();
 			audioRing.push(value);
 			SDL_UnlockAudio();
+			if (ffmpeg_audio) fwrite(&value, 1, 1, ffmpeg_audio);
 		}
 	}
 
@@ -1377,16 +1384,16 @@ bool avr8::init_gui()
 	init_joysticks();
 
 	if (fullscreen)
-		screen = SDL_SetVideoMode(800,600,32,sdl_flags | SDL_FULLSCREEN);
+		screen = SDL_SetVideoMode(640,480,32,sdl_flags | SDL_FULLSCREEN);
 	else
-		screen = SDL_SetVideoMode(630,448,32,sdl_flags);
+		screen = SDL_SetVideoMode(640,480,32,sdl_flags);
 	if (!screen)
 	{
-		fprintf(stderr, "Unable to set 630x448x32 video mode.\n");
+		fprintf(stderr, "Unable to set 640x480x32 video mode.\n");
 		return false;
 	}
-	else if (fullscreen)	// Center in fullscreen
-		inset = ((600-448)/2) * screen->pitch + 4 * ((800-630)/2);
+	inset = ((480-448)/2) * screen->pitch + 4 * ((640-630)/2); // Center
 
 	if (SDL_MUSTLOCK(screen) && SDL_LockSurface(screen) < 0)
 		return false;
@@ -2500,6 +2507,26 @@ void avr8::shutdown(int errcode){
     	fclose(captureFile);
     }
 
+    bool combine = (ffmpeg_video != 0) && (ffmpeg_audio != 0);
+    if (ffmpeg_video) {
+        pclose(ffmpeg_video);
+        ffmpeg_video = 0;
+    }
+    if (ffmpeg_audio) {
+        pclose(ffmpeg_audio);
+        ffmpeg_audio = 0;
+    }
+    if (combine) {
+        FILE* ffmpeg_mux = popen("avconv -y -i test.mp4 -i test.mp3 -vcodec copy -acodec copy -f mp4 uzem.mp4", "r");
+        if (ffmpeg_mux) {
+	    SDL_Quit();
+            pclose(ffmpeg_mux);
+            ffmpeg_mux = 0;
+	    unlink("test.mp4");
+            unlink("test.mp3");
+        }
+    }
+
 #if GUI
 	if (joystickFile) {
 		FILE* f = fopen(joystickFile,"wb");
diff --git a/tools/uzem/avr8.h b/tools/uzem/avr8.h
index 0a5e674..8d87910 100644
--- a/tools/uzem/avr8.h
+++ b/tools/uzem/avr8.h
@@ -289,7 +289,7 @@ struct avr8
 		enableSound(true), fullscreen(false), interlaced(false), lastFlip(0), inset(0), prevPortB(0), 
 		prevWDR(0), frameCounter(0),  new_input_mode(false),gdb(0),enableGdb(false), SDpath(NULL), gdbBreakpointFound(false),gdbInvalidOpcode(false),gdbPort(1284),state(CPU_STOPPED),
         spiByte(0), spiClock(0), spiTransfer(0), spiState(SPI_IDLE_STATE), spiResponsePtr(0), spiResponseEnd(0),eepromFile("eeprom.bin"),joystickFile(0),captureFile(NULL),
-		captureMode(CAPTURE_NONE),watchdogTimer(0),
+		captureMode(CAPTURE_NONE),ffmpeg_video(0),ffmpeg_audio(0),watchdogTimer(0),
 
 
     #if defined(__WIN32__)
@@ -422,6 +422,9 @@ struct avr8
     long captureSize;
     long capturePtr;
 
+    FILE* ffmpeg_video;
+    FILE* ffmpeg_audio;
+
     FILE* sdImage;
     u8* emulatedMBR;
     u32 emulatedReadPos;
diff --git a/tools/uzem/uzem.cpp b/tools/uzem/uzem.cpp
index b2a0873..0884096 100644
--- a/tools/uzem/uzem.cpp
+++ b/tools/uzem/uzem.cpp
@@ -61,6 +49,7 @@ static const struct option longopts[] ={
     { "port"       , required_argument, NULL, 't' },
     { "capture"    , no_argument,       NULL, 'c' },
     { "loadcap"    , no_argument,       NULL, 'l' },
+    { "record"     , no_argument,       NULL, 'o' },
 
 #if defined(__WIN32__)
     { "sd"         , required_argument, NULL, 's' },
@@ -68,7 +57,7 @@ static const struct option longopts[] ={
     {NULL          , 0                , NULL, 0}
 };
 
-   static const char* shortopts = "hnfclwxim2re:p:bdt:k:s:v";
+   static const char* shortopts = "hnfclowxim2re:p:bdt:k:s:v";
 
 #define printerr(fmt,...) fprintf(stderr,fmt,##__VA_ARGS__)
 
@@ -95,6 +84,7 @@ void showHelp(char* programName){
     printerr("\t--port -t <port>    Port used by gdb (default 1284).\n");
     printerr("\t--capture -c        Captures controllers data to file.\n");
     printerr("\t--loadcap -l        Load and replays controllers data from file.\n");
+    printerr("\t--record -o         Records a movie of the emulator's output to file.\n");
 }
 
 char *strlwr(char *str)
@@ -201,6 +191,10 @@ int main(int argc,char **argv)
         case 'l':
             uzebox.captureMode=CAPTURE_READ;
             break;
+        case 'o':
+            uzebox.ffmpeg_video = popen("avconv -y -f rawvideo -s 640x480 -pix_fmt bgra -r 60 -i - -vf scale=-1:720 -sws_flags neighbor -an -b:v 1000k test.mp4", "w");
+            uzebox.ffmpeg_audio = popen("avconv -y -f u8 -ar 15700 -ac 1 -i - -acodec libmp3lame -ar 44.1k test.mp3", "w");
+            break;
         case 'd':
             uzebox.enableGdb = true;
             break;
Edit: And don't change the sample rate to 15734 in the audio line, because that will desynchronize the audio, keep it 15700, because that's what you configured SDL to use.
Post Reply