#import "GPUImageVideoCamera.h" #import "GPUImageMovieWriter.h" #import "GPUImageFilter.h" // Color Conversion Constants (YUV to RGB) including adjustment from 16-235/16-240 (video range) // BT.601, which is the standard for SDTV. const GLfloat kColorConversion601[] = { // 1.164, 1.164, 1.164, // 0.0, -0.392, 2.017, // 1.596, -0.813, 0.0, 1, 1, 1, 0, -.39465, 2.03211, 1.13983, -.58060, 0, }; // BT.709, which is the standard for HDTV. const GLfloat kColorConversion709[] = { // 1.164, 1.164, 1.164, // 0.0, -0.213, 2.112, // 1.793, -0.533, 0.0, 1, 1, 1, 0, -.21482, 2.12798, 1.28033, -.38059, 0, }; // BT.601 full range (ref: http://www.equasys.de/colorconversion.html) const GLfloat kColorConversion601FullRange[] = { 1.0, 1.0, 1.0, 0.0, -0.343, 1.765, 1.4, -0.711, 0.0, }; NSString *const kGPUImageYUVVideoRangeConversionForRGFragmentShaderString = SHADER_STRING ( varying highp vec2 textureCoordinate; uniform sampler2D luminanceTexture; uniform sampler2D chrominanceTexture; uniform mediump mat3 colorConversionMatrix; void main() { mediump vec3 yuv; lowp vec3 rgb; yuv.x = texture2D(luminanceTexture, textureCoordinate).r; yuv.yz = texture2D(chrominanceTexture, textureCoordinate).rg - vec2(0.5, 0.5); rgb = colorConversionMatrix * yuv; gl_FragColor = vec4(rgb, 1); } ); NSString *const kGPUImageYUVFullRangeConversionForLAFragmentShaderString = SHADER_STRING ( varying highp vec2 textureCoordinate; uniform sampler2D luminanceTexture; uniform sampler2D chrominanceTexture; uniform mediump mat3 colorConversionMatrix; void main() { mediump vec3 yuv; lowp vec3 rgb; yuv.x = texture2D(luminanceTexture, textureCoordinate).r; yuv.yz = texture2D(chrominanceTexture, textureCoordinate).ra - vec2(0.5, 0.5); rgb = colorConversionMatrix * yuv; gl_FragColor = vec4(rgb, 1); } ); NSString *const kGPUImageYUVVideoRangeConversionForLAFragmentShaderString = SHADER_STRING ( varying highp vec2 textureCoordinate; uniform sampler2D luminanceTexture; uniform sampler2D chrominanceTexture; uniform mediump mat3 colorConversionMatrix; void main() { mediump vec3 yuv; lowp vec3 rgb; yuv.x = texture2D(luminanceTexture, textureCoordinate).r - (16.0/255.0); yuv.yz = texture2D(chrominanceTexture, textureCoordinate).ra - vec2(0.5, 0.5); rgb = colorConversionMatrix * yuv; gl_FragColor = vec4(rgb, 1); } ); #pragma mark - #pragma mark Private methods and instance variables @interface GPUImageVideoCamera () { AVCaptureDeviceInput *audioInput; AVCaptureAudioDataOutput *audioOutput; NSDate *startingCaptureTime; dispatch_queue_t cameraProcessingQueue, audioProcessingQueue; GLProgram *yuvConversionProgram; GLint yuvConversionPositionAttribute, yuvConversionTextureCoordinateAttribute; GLint yuvConversionLuminanceTextureUniform, yuvConversionChrominanceTextureUniform; GLint yuvConversionMatrixUniform; const GLfloat *_preferredConversion; BOOL isFullYUVRange; int imageBufferWidth, imageBufferHeight; BOOL addedAudioInputsDueToEncodingTarget; } - (void)updateOrientationSendToTargets; - (void)convertYUVToRGBOutput; @end @implementation GPUImageVideoCamera @synthesize captureSessionPreset = _captureSessionPreset; @synthesize captureSession = _captureSession; @synthesize inputCamera = _inputCamera; @synthesize runBenchmark = _runBenchmark; @synthesize outputImageOrientation = _outputImageOrientation; @synthesize delegate = _delegate; @synthesize horizontallyMirrorFrontFacingCamera = _horizontallyMirrorFrontFacingCamera, horizontallyMirrorRearFacingCamera = _horizontallyMirrorRearFacingCamera; @synthesize frameRate = _frameRate; #pragma mark - #pragma mark Initialization and teardown - (id)init; { if (!(self = [self initWithSessionPreset:AVCaptureSessionPreset640x480 cameraPosition:AVCaptureDevicePositionBack])) { return nil; } return self; } - (id)initWithSessionPreset:(NSString *)sessionPreset cameraPosition:(AVCaptureDevicePosition)cameraPosition; { if (!(self = [super init])) { return nil; } cameraProcessingQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH,0); audioProcessingQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW,0); frameRenderingSemaphore = dispatch_semaphore_create(1); _frameRate = 0; // This will not set frame rate unless this value gets set to 1 or above _runBenchmark = NO; capturePaused = NO; outputRotation = kGPUImageNoRotation; internalRotation = kGPUImageNoRotation; captureAsYUV = YES; _preferredConversion = kColorConversion709; // Grab the back-facing or front-facing camera _inputCamera = nil; NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in devices) { if ([device position] == cameraPosition) { _inputCamera = device; } } if (!_inputCamera) { return nil; } // Create the capture session _captureSession = [[AVCaptureSession alloc] init]; [_captureSession beginConfiguration]; // Add the video input NSError *error = nil; videoInput = [[AVCaptureDeviceInput alloc] initWithDevice:_inputCamera error:&error]; if ([_captureSession canAddInput:videoInput]) { [_captureSession addInput:videoInput]; } // Add the video frame output videoOutput = [[AVCaptureVideoDataOutput alloc] init]; [videoOutput setAlwaysDiscardsLateVideoFrames:NO]; // if (captureAsYUV && [GPUImageContext deviceSupportsRedTextures]) if (captureAsYUV && [GPUImageContext supportsFastTextureUpload]) { BOOL supportsFullYUVRange = NO; NSArray *supportedPixelFormats = videoOutput.availableVideoCVPixelFormatTypes; for (NSNumber *currentPixelFormat in supportedPixelFormats) { if ([currentPixelFormat intValue] == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { supportsFullYUVRange = YES; } } if (supportsFullYUVRange) { [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; isFullYUVRange = YES; } else { [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; isFullYUVRange = NO; } } else { [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; } runSynchronouslyOnVideoProcessingQueue(^{ if (captureAsYUV) { [GPUImageContext useImageProcessingContext]; // if ([GPUImageContext deviceSupportsRedTextures]) // { // yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForRGFragmentShaderString]; // } // else // { if (isFullYUVRange) { yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString]; } else { yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForLAFragmentShaderString]; } // } if (!yuvConversionProgram.initialized) { [yuvConversionProgram addAttribute:@"position"]; [yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; if (![yuvConversionProgram link]) { NSString *progLog = [yuvConversionProgram programLog]; NSLog(@"Program link log: %@", progLog); NSString *fragLog = [yuvConversionProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); NSString *vertLog = [yuvConversionProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); yuvConversionProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } yuvConversionPositionAttribute = [yuvConversionProgram attributeIndex:@"position"]; yuvConversionTextureCoordinateAttribute = [yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; yuvConversionLuminanceTextureUniform = [yuvConversionProgram uniformIndex:@"luminanceTexture"]; yuvConversionChrominanceTextureUniform = [yuvConversionProgram uniformIndex:@"chrominanceTexture"]; yuvConversionMatrixUniform = [yuvConversionProgram uniformIndex:@"colorConversionMatrix"]; [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; glEnableVertexAttribArray(yuvConversionPositionAttribute); glEnableVertexAttribArray(yuvConversionTextureCoordinateAttribute); } }); [videoOutput setSampleBufferDelegate:self queue:cameraProcessingQueue]; if ([_captureSession canAddOutput:videoOutput]) { [_captureSession addOutput:videoOutput]; } else { NSLog(@"Couldn't add video output"); return nil; } _captureSessionPreset = sessionPreset; [_captureSession setSessionPreset:_captureSessionPreset]; // This will let you get 60 FPS video from the 720p preset on an iPhone 4S, but only that device and that preset // AVCaptureConnection *conn = [videoOutput connectionWithMediaType:AVMediaTypeVideo]; // // if (conn.supportsVideoMinFrameDuration) // conn.videoMinFrameDuration = CMTimeMake(1,60); // if (conn.supportsVideoMaxFrameDuration) // conn.videoMaxFrameDuration = CMTimeMake(1,60); [_captureSession commitConfiguration]; return self; } - (GPUImageFramebuffer *)framebufferForOutput; { return outputFramebuffer; } - (void)dealloc { [self stopCameraCapture]; [videoOutput setSampleBufferDelegate:nil queue:dispatch_get_main_queue()]; [audioOutput setSampleBufferDelegate:nil queue:dispatch_get_main_queue()]; [self removeInputsAndOutputs]; // ARC forbids explicit message send of 'release'; since iOS 6 even for dispatch_release() calls: stripping it out in that case is required. #if ( (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_6_0) || (!defined(__IPHONE_6_0)) ) if (frameRenderingSemaphore != NULL) { dispatch_release(frameRenderingSemaphore); } #endif } - (BOOL)addAudioInputsAndOutputs { if (audioOutput) return NO; [_captureSession beginConfiguration]; _microphone = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; audioInput = [AVCaptureDeviceInput deviceInputWithDevice:_microphone error:nil]; if ([_captureSession canAddInput:audioInput]) { [_captureSession addInput:audioInput]; } audioOutput = [[AVCaptureAudioDataOutput alloc] init]; if ([_captureSession canAddOutput:audioOutput]) { [_captureSession addOutput:audioOutput]; } else { NSLog(@"Couldn't add audio output"); } [audioOutput setSampleBufferDelegate:self queue:audioProcessingQueue]; [_captureSession commitConfiguration]; return YES; } - (BOOL)removeAudioInputsAndOutputs { if (!audioOutput) return NO; [_captureSession beginConfiguration]; [_captureSession removeInput:audioInput]; [_captureSession removeOutput:audioOutput]; audioInput = nil; audioOutput = nil; _microphone = nil; [_captureSession commitConfiguration]; return YES; } - (void)removeInputsAndOutputs; { [_captureSession beginConfiguration]; if (videoInput) { [_captureSession removeInput:videoInput]; [_captureSession removeOutput:videoOutput]; videoInput = nil; videoOutput = nil; } if (_microphone != nil) { [_captureSession removeInput:audioInput]; [_captureSession removeOutput:audioOutput]; audioInput = nil; audioOutput = nil; _microphone = nil; } [_captureSession commitConfiguration]; } #pragma mark - #pragma mark Managing targets - (void)addTarget:(id)newTarget atTextureLocation:(NSInteger)textureLocation; { [super addTarget:newTarget atTextureLocation:textureLocation]; [newTarget setInputRotation:outputRotation atIndex:textureLocation]; } #pragma mark - #pragma mark Manage the camera video stream - (void)startCameraCapture; { if (![_captureSession isRunning]) { startingCaptureTime = [NSDate date]; [_captureSession startRunning]; }; } - (void)stopCameraCapture; { if ([_captureSession isRunning]) { [_captureSession stopRunning]; } } - (void)pauseCameraCapture; { capturePaused = YES; } - (void)resumeCameraCapture; { capturePaused = NO; } - (void)rotateCamera { if (self.frontFacingCameraPresent == NO) return; NSError *error; AVCaptureDeviceInput *newVideoInput; AVCaptureDevicePosition currentCameraPosition = [[videoInput device] position]; if (currentCameraPosition == AVCaptureDevicePositionBack) { currentCameraPosition = AVCaptureDevicePositionFront; } else { currentCameraPosition = AVCaptureDevicePositionBack; } AVCaptureDevice *backFacingCamera = nil; NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in devices) { if ([device position] == currentCameraPosition) { backFacingCamera = device; } } newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:backFacingCamera error:&error]; if (newVideoInput != nil) { [_captureSession beginConfiguration]; [_captureSession removeInput:videoInput]; if ([_captureSession canAddInput:newVideoInput]) { [_captureSession addInput:newVideoInput]; videoInput = newVideoInput; } else { [_captureSession addInput:videoInput]; } //captureSession.sessionPreset = oriPreset; [_captureSession commitConfiguration]; } _inputCamera = backFacingCamera; [self setOutputImageOrientation:_outputImageOrientation]; } - (AVCaptureDevicePosition)cameraPosition { return [[videoInput device] position]; } + (BOOL)isBackFacingCameraPresent; { NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in devices) { if ([device position] == AVCaptureDevicePositionBack) return YES; } return NO; } - (BOOL)isBackFacingCameraPresent { return [GPUImageVideoCamera isBackFacingCameraPresent]; } + (BOOL)isFrontFacingCameraPresent; { NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in devices) { if ([device position] == AVCaptureDevicePositionFront) return YES; } return NO; } - (BOOL)isFrontFacingCameraPresent { return [GPUImageVideoCamera isFrontFacingCameraPresent]; } - (void)setCaptureSessionPreset:(NSString *)captureSessionPreset; { [_captureSession beginConfiguration]; _captureSessionPreset = captureSessionPreset; [_captureSession setSessionPreset:_captureSessionPreset]; [_captureSession commitConfiguration]; } - (void)setFrameRate:(int32_t)frameRate; { _frameRate = frameRate; if (_frameRate > 0) { if ([_inputCamera respondsToSelector:@selector(setActiveVideoMinFrameDuration:)] && [_inputCamera respondsToSelector:@selector(setActiveVideoMaxFrameDuration:)]) { NSError *error; [_inputCamera lockForConfiguration:&error]; if (error == nil) { #if defined(__IPHONE_7_0) [_inputCamera setActiveVideoMinFrameDuration:CMTimeMake(1, _frameRate)]; [_inputCamera setActiveVideoMaxFrameDuration:CMTimeMake(1, _frameRate)]; #endif } [_inputCamera unlockForConfiguration]; } else { for (AVCaptureConnection *connection in videoOutput.connections) { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" if ([connection respondsToSelector:@selector(setVideoMinFrameDuration:)]) connection.videoMinFrameDuration = CMTimeMake(1, _frameRate); if ([connection respondsToSelector:@selector(setVideoMaxFrameDuration:)]) connection.videoMaxFrameDuration = CMTimeMake(1, _frameRate); #pragma clang diagnostic pop } } } else { if ([_inputCamera respondsToSelector:@selector(setActiveVideoMinFrameDuration:)] && [_inputCamera respondsToSelector:@selector(setActiveVideoMaxFrameDuration:)]) { NSError *error; [_inputCamera lockForConfiguration:&error]; if (error == nil) { #if defined(__IPHONE_7_0) [_inputCamera setActiveVideoMinFrameDuration:kCMTimeInvalid]; [_inputCamera setActiveVideoMaxFrameDuration:kCMTimeInvalid]; #endif } [_inputCamera unlockForConfiguration]; } else { for (AVCaptureConnection *connection in videoOutput.connections) { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" if ([connection respondsToSelector:@selector(setVideoMinFrameDuration:)]) connection.videoMinFrameDuration = kCMTimeInvalid; // This sets videoMinFrameDuration back to default if ([connection respondsToSelector:@selector(setVideoMaxFrameDuration:)]) connection.videoMaxFrameDuration = kCMTimeInvalid; // This sets videoMaxFrameDuration back to default #pragma clang diagnostic pop } } } } - (int32_t)frameRate; { return _frameRate; } - (AVCaptureConnection *)videoCaptureConnection { for (AVCaptureConnection *connection in [videoOutput connections] ) { for ( AVCaptureInputPort *port in [connection inputPorts] ) { if ( [[port mediaType] isEqual:AVMediaTypeVideo] ) { return connection; } } } return nil; } #define INITIALFRAMESTOIGNOREFORBENCHMARK 5 - (void)updateTargetsForVideoCameraUsingCacheTextureAtWidth:(int)bufferWidth height:(int)bufferHeight time:(CMTime)currentTime; { // First, update all the framebuffers in the targets for (id currentTarget in targets) { if ([currentTarget enabled]) { NSInteger indexOfObject = [targets indexOfObject:currentTarget]; NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; if (currentTarget != self.targetToIgnoreForUpdates) { [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget]; [currentTarget setInputSize:CGSizeMake(bufferWidth, bufferHeight) atIndex:textureIndexOfTarget]; if ([currentTarget wantsMonochromeInput] && captureAsYUV) { [currentTarget setCurrentlyReceivingMonochromeInput:YES]; // TODO: Replace optimization for monochrome output [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; } else { [currentTarget setCurrentlyReceivingMonochromeInput:NO]; [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; } } else { [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget]; [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; } } } // Then release our hold on the local framebuffer to send it back to the cache as soon as it's no longer needed [outputFramebuffer unlock]; outputFramebuffer = nil; // Finally, trigger rendering as needed for (id currentTarget in targets) { if ([currentTarget enabled]) { NSInteger indexOfObject = [targets indexOfObject:currentTarget]; NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; if (currentTarget != self.targetToIgnoreForUpdates) { [currentTarget newFrameReadyAtTime:currentTime atIndex:textureIndexOfTarget]; } } } } - (void)processVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer; { if (capturePaused) { return; } CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent(); CVImageBufferRef cameraFrame = CMSampleBufferGetImageBuffer(sampleBuffer); int bufferWidth = (int) CVPixelBufferGetWidth(cameraFrame); int bufferHeight = (int) CVPixelBufferGetHeight(cameraFrame); CFTypeRef colorAttachments = CVBufferGetAttachment(cameraFrame, kCVImageBufferYCbCrMatrixKey, NULL); if (colorAttachments != NULL) { if(CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) { if (isFullYUVRange) { _preferredConversion = kColorConversion601FullRange; } else { _preferredConversion = kColorConversion601; } } else { _preferredConversion = kColorConversion709; } } else { if (isFullYUVRange) { _preferredConversion = kColorConversion601FullRange; } else { _preferredConversion = kColorConversion601; } } CMTime currentTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); [GPUImageContext useImageProcessingContext]; if ([GPUImageContext supportsFastTextureUpload] && captureAsYUV) { CVOpenGLESTextureRef luminanceTextureRef = NULL; CVOpenGLESTextureRef chrominanceTextureRef = NULL; // if (captureAsYUV && [GPUImageContext deviceSupportsRedTextures]) if (CVPixelBufferGetPlaneCount(cameraFrame) > 0) // Check for YUV planar inputs to do RGB conversion { CVPixelBufferLockBaseAddress(cameraFrame, 0); if ( (imageBufferWidth != bufferWidth) && (imageBufferHeight != bufferHeight) ) { imageBufferWidth = bufferWidth; imageBufferHeight = bufferHeight; } CVReturn err; // Y-plane glActiveTexture(GL_TEXTURE4); if ([GPUImageContext deviceSupportsRedTextures]) { // err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, coreVideoTextureCache, cameraFrame, NULL, GL_TEXTURE_2D, GL_RED_EXT, bufferWidth, bufferHeight, GL_RED_EXT, GL_UNSIGNED_BYTE, 0, &luminanceTextureRef); err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE, bufferWidth, bufferHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &luminanceTextureRef); } else { err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE, bufferWidth, bufferHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &luminanceTextureRef); } if (err) { NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err); } luminanceTexture = CVOpenGLESTextureGetName(luminanceTextureRef); glBindTexture(GL_TEXTURE_2D, luminanceTexture); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // UV-plane glActiveTexture(GL_TEXTURE5); if ([GPUImageContext deviceSupportsRedTextures]) { // err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, coreVideoTextureCache, cameraFrame, NULL, GL_TEXTURE_2D, GL_RG_EXT, bufferWidth/2, bufferHeight/2, GL_RG_EXT, GL_UNSIGNED_BYTE, 1, &chrominanceTextureRef); err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, bufferWidth/2, bufferHeight/2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &chrominanceTextureRef); } else { err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, bufferWidth/2, bufferHeight/2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &chrominanceTextureRef); } if (err) { NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err); } chrominanceTexture = CVOpenGLESTextureGetName(chrominanceTextureRef); glBindTexture(GL_TEXTURE_2D, chrominanceTexture); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // if (!allTargetsWantMonochromeData) // { [self convertYUVToRGBOutput]; // } int rotatedImageBufferWidth = bufferWidth, rotatedImageBufferHeight = bufferHeight; if (GPUImageRotationSwapsWidthAndHeight(internalRotation)) { rotatedImageBufferWidth = bufferHeight; rotatedImageBufferHeight = bufferWidth; } [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:rotatedImageBufferWidth height:rotatedImageBufferHeight time:currentTime]; CVPixelBufferUnlockBaseAddress(cameraFrame, 0); CFRelease(luminanceTextureRef); CFRelease(chrominanceTextureRef); } else { // TODO: Mesh this with the output framebuffer structure // CVPixelBufferLockBaseAddress(cameraFrame, 0); // // CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_RGBA, bufferWidth, bufferHeight, GL_BGRA, GL_UNSIGNED_BYTE, 0, &texture); // // if (!texture || err) { // NSLog(@"Camera CVOpenGLESTextureCacheCreateTextureFromImage failed (error: %d)", err); // NSAssert(NO, @"Camera failure"); // return; // } // // outputTexture = CVOpenGLESTextureGetName(texture); // // glBindTexture(CVOpenGLESTextureGetTarget(texture), outputTexture); // glBindTexture(GL_TEXTURE_2D, outputTexture); // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // // [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:bufferWidth height:bufferHeight time:currentTime]; // // CVPixelBufferUnlockBaseAddress(cameraFrame, 0); // CFRelease(texture); // // outputTexture = 0; } if (_runBenchmark) { numberOfFramesCaptured++; if (numberOfFramesCaptured > INITIALFRAMESTOIGNOREFORBENCHMARK) { CFAbsoluteTime currentFrameTime = (CFAbsoluteTimeGetCurrent() - startTime); totalFrameTimeDuringCapture += currentFrameTime; NSLog(@"Average frame time : %f ms", [self averageFrameDurationDuringCapture]); NSLog(@"Current frame time : %f ms", 1000.0 * currentFrameTime); } } } else { CVPixelBufferLockBaseAddress(cameraFrame, 0); int bytesPerRow = (int) CVPixelBufferGetBytesPerRow(cameraFrame); outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(bytesPerRow / 4, bufferHeight) onlyTexture:YES]; [outputFramebuffer activateFramebuffer]; glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]); // glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bufferWidth, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame)); // Using BGRA extension to pull in video frame data directly // The use of bytesPerRow / 4 accounts for a display glitch present in preview video frames when using the photo preset on the camera glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bytesPerRow / 4, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame)); [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:bytesPerRow / 4 height:bufferHeight time:currentTime]; CVPixelBufferUnlockBaseAddress(cameraFrame, 0); if (_runBenchmark) { numberOfFramesCaptured++; if (numberOfFramesCaptured > INITIALFRAMESTOIGNOREFORBENCHMARK) { CFAbsoluteTime currentFrameTime = (CFAbsoluteTimeGetCurrent() - startTime); totalFrameTimeDuringCapture += currentFrameTime; } } } } - (void)processAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer; { [self.audioEncodingTarget processAudioBuffer:sampleBuffer]; } - (void)convertYUVToRGBOutput; { [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; int rotatedImageBufferWidth = imageBufferWidth, rotatedImageBufferHeight = imageBufferHeight; if (GPUImageRotationSwapsWidthAndHeight(internalRotation)) { rotatedImageBufferWidth = imageBufferHeight; rotatedImageBufferHeight = imageBufferWidth; } outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(rotatedImageBufferWidth, rotatedImageBufferHeight) textureOptions:self.outputTextureOptions onlyTexture:NO]; [outputFramebuffer activateFramebuffer]; glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); static const GLfloat squareVertices[] = { -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, }; glActiveTexture(GL_TEXTURE4); glBindTexture(GL_TEXTURE_2D, luminanceTexture); glUniform1i(yuvConversionLuminanceTextureUniform, 4); glActiveTexture(GL_TEXTURE5); glBindTexture(GL_TEXTURE_2D, chrominanceTexture); glUniform1i(yuvConversionChrominanceTextureUniform, 5); glUniformMatrix3fv(yuvConversionMatrixUniform, 1, GL_FALSE, _preferredConversion); glVertexAttribPointer(yuvConversionPositionAttribute, 2, GL_FLOAT, 0, 0, squareVertices); glVertexAttribPointer(yuvConversionTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, [GPUImageFilter textureCoordinatesForRotation:internalRotation]); glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); } #pragma mark - #pragma mark Benchmarking - (CGFloat)averageFrameDurationDuringCapture; { return (totalFrameTimeDuringCapture / (CGFloat)(numberOfFramesCaptured - INITIALFRAMESTOIGNOREFORBENCHMARK)) * 1000.0; } - (void)resetBenchmarkAverage; { numberOfFramesCaptured = 0; totalFrameTimeDuringCapture = 0.0; } #pragma mark - #pragma mark AVCaptureVideoDataOutputSampleBufferDelegate - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { if (!self.captureSession.isRunning) { return; } else if (captureOutput == audioOutput) { [self processAudioSampleBuffer:sampleBuffer]; } else { if (dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_NOW) != 0) { return; } CFRetain(sampleBuffer); runAsynchronouslyOnVideoProcessingQueue(^{ //Feature Detection Hook. if (self.delegate) { [self.delegate willOutputSampleBuffer:sampleBuffer]; } [self processVideoSampleBuffer:sampleBuffer]; CFRelease(sampleBuffer); dispatch_semaphore_signal(frameRenderingSemaphore); }); } } #pragma mark - #pragma mark Accessors - (void)setAudioEncodingTarget:(GPUImageMovieWriter *)newValue; { if (newValue) { /* Add audio inputs and outputs, if necessary */ addedAudioInputsDueToEncodingTarget |= [self addAudioInputsAndOutputs]; } else if (addedAudioInputsDueToEncodingTarget) { /* Remove audio inputs and outputs, if they were added by previously setting the audio encoding target */ [self removeAudioInputsAndOutputs]; addedAudioInputsDueToEncodingTarget = NO; } [super setAudioEncodingTarget:newValue]; } - (void)updateOrientationSendToTargets; { runSynchronouslyOnVideoProcessingQueue(^{ // From the iOS 5.0 release notes: // In previous iOS versions, the front-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeLeft and the back-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeRight. if (captureAsYUV && [GPUImageContext supportsFastTextureUpload]) { outputRotation = kGPUImageNoRotation; if ([self cameraPosition] == AVCaptureDevicePositionBack) { if (_horizontallyMirrorRearFacingCamera) { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRightFlipVertical; break; case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotate180; break; case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageFlipHorizonal; break; case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageFlipVertical; break; default:internalRotation = kGPUImageNoRotation; } } else { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRight; break; case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateLeft; break; case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageRotate180; break; case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageNoRotation; break; default:internalRotation = kGPUImageNoRotation; } } } else { if (_horizontallyMirrorFrontFacingCamera) { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRightFlipVertical; break; case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateRightFlipHorizontal; break; case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageFlipHorizonal; break; case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageFlipVertical; break; default:internalRotation = kGPUImageNoRotation; } } else { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRight; break; case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateLeft; break; case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageNoRotation; break; case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageRotate180; break; default:internalRotation = kGPUImageNoRotation; } } } } else { if ([self cameraPosition] == AVCaptureDevicePositionBack) { if (_horizontallyMirrorRearFacingCamera) { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRightFlipVertical; break; case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotate180; break; case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageFlipHorizonal; break; case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageFlipVertical; break; default:outputRotation = kGPUImageNoRotation; } } else { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRight; break; case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateLeft; break; case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageRotate180; break; case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageNoRotation; break; default:outputRotation = kGPUImageNoRotation; } } } else { if (_horizontallyMirrorFrontFacingCamera) { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRightFlipVertical; break; case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateRightFlipHorizontal; break; case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageFlipHorizonal; break; case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageFlipVertical; break; default:outputRotation = kGPUImageNoRotation; } } else { switch(_outputImageOrientation) { case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRight; break; case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateLeft; break; case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageNoRotation; break; case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageRotate180; break; default:outputRotation = kGPUImageNoRotation; } } } } for (id currentTarget in targets) { NSInteger indexOfObject = [targets indexOfObject:currentTarget]; [currentTarget setInputRotation:outputRotation atIndex:[[targetTextureIndices objectAtIndex:indexOfObject] integerValue]]; } }); } - (void)setOutputImageOrientation:(UIInterfaceOrientation)newValue; { _outputImageOrientation = newValue; [self updateOrientationSendToTargets]; } - (void)setHorizontallyMirrorFrontFacingCamera:(BOOL)newValue { _horizontallyMirrorFrontFacingCamera = newValue; [self updateOrientationSendToTargets]; } - (void)setHorizontallyMirrorRearFacingCamera:(BOOL)newValue { _horizontallyMirrorRearFacingCamera = newValue; [self updateOrientationSendToTargets]; } @end