使用OpenGL实时预览CameraX捕捉到的摄像头数据

CameraX 是一个 Jetpack 支持库&＃xff0c;旨在帮助您简化相机应用的开发工作。笔者看了下网上关于CameraX的资料虽然很多&＃xff0c;但是很多基本上都是官网资料的翻版&＃xff0c;学习的价值很没有直接看官网的高。

也有些博客介绍了CameraX结合OpenGL渲染的的例子&＃xff0c;但好像都建立在Preview类的setOnPreviewOutputUpdateListener这个方法中进行处理&＃xff0c;但是笔者更新CameraX版本之后发现setOnPreviewOutputUpdateListener这个
方法直接没了&＃xff0c;完犊子了…

当然本文所介绍的方法随着CameraX的发展也会过时&＃xff0c;但也希望能起到一点抛砖引玉的作用。。。。

首先自定义一个OpenGL的渲染View&＃xff0c;继承于GLSurfaceView&＃xff0c;GLCameraView.java:


public class GLCameraView extends GLSurfaceView implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {private static final String LOG_TAG &＃61; "OpenGLCameraX";private Executor executor &＃61; Executors.newSingleThreadExecutor();private int textureId;private SurfaceTexture surfaceTexture;private int vPosition;private int vCoord;private int programId;private int textureMatrixId;private float[] textureMatrix &＃61; new float[16];protected FloatBuffer mGLVertexBuffer;protected FloatBuffer mGLTextureBuffer;public GLCameraView(Context context) {this(context, null);}public GLCameraView(Context context, AttributeSet attrs) {super(context, attrs);setEGLContextClientVersion(2);setRenderer(this);// 设置非连续渲染setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);}&＃64;SuppressLint("UnsafeExperimentalUsageError")public void attachPreview(Preview preview) {preview.setSurfaceProvider(new Preview.SurfaceProvider() {&＃64;Overridepublic void onSurfaceRequested(&＃64;NonNull SurfaceRequest request) {Surface surface &＃61; new Surface(surfaceTexture);request.provideSurface(surface, executor, new Consumer() {&＃64;Overridepublic void accept(SurfaceRequest.Result result) {surface.release();surfaceTexture.release();Log.v(LOG_TAG, "--accept------");}});}});}&＃64;Overridepublic void onSurfaceCreated(GL10 gl, EGLConfig config) {int[] ids &＃61; new int[1];// OpenGL相关GLES20.glGenTextures(1, ids, 0);textureId &＃61; ids[0];surfaceTexture &＃61; new SurfaceTexture(textureId);surfaceTexture.setOnFrameAvailableListener(this::onFrameAvailable);String vertexShader &＃61; OpenGLUtils.readRawTextFile(getContext(), R.raw.camera_vertex);String fragmentShader &＃61; OpenGLUtils.readRawTextFile(getContext(), R.raw.camera_frag);programId &＃61; OpenGLUtils.loadProgram(vertexShader, fragmentShader);vPosition &＃61; GLES20.glGetAttribLocation(programId, "vPosition");vCoord &＃61; GLES20.glGetAttribLocation(programId, "vCoord");textureMatrixId &＃61; GLES20.glGetUniformLocation(programId, "textureMatrix");// 4个顶点&＃xff0c;每个顶点有两个浮点型&＃xff0c;每个浮点型占4个字节mGLVertexBuffer &＃61; ByteBuffer.allocateDirect(4 * 4 * 2).order(ByteOrder.nativeOrder()).asFloatBuffer();mGLVertexBuffer.clear();// 顶点坐标float[] VERTEX &＃61; {-1.0f, -1.0f,1.0f, -1.0f,-1.0f, 1.0f,1.0f, 1.0f};mGLVertexBuffer.put(VERTEX);// 纹理坐标mGLTextureBuffer &＃61; ByteBuffer.allocateDirect(4 * 2 * 4).order(ByteOrder.nativeOrder()).asFloatBuffer();mGLTextureBuffer.clear();// 正常的纹理贴图坐标&＃xff0c;但是贴出的图是上下颠倒的&＃xff0c;所以需要修改一下
// float[] TEXTURE &＃61; {
// 0.0f, 1.0f,
// 1.0f, 1.0f,
// 0.0f, 0.0f,
// 1.0f, 0.0f
// };// 修复上下颠倒后的纹理贴图坐标float[] TEXTURE &＃61; {0.0f, 0.0f,1.0f, 0.0f,0.0f, 1.0f,1.0f, 1.0f};mGLTextureBuffer.put(TEXTURE);}&＃64;Overridepublic void onSurfaceChanged(GL10 gl, int width, int height) {GLES20.glViewport(0, 0, width, height);}&＃64;Overridepublic void onDrawFrame(GL10 gl) {// 清屏GLES20.glClearColor(1, 0, 0, 0);GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);// 更新纹理surfaceTexture.updateTexImage();surfaceTexture.getTransformMatrix(textureMatrix);GLES20.glUseProgram(programId);//变换矩阵GLES20.glUniformMatrix4fv(textureMatrixId, 1, false, textureMatrix, 0);// 传递坐标数据mGLVertexBuffer.position(0);GLES20.glVertexAttribPointer(vPosition, 2, GLES20.GL_FLOAT, false, 0, mGLVertexBuffer);GLES20.glEnableVertexAttribArray(vPosition);// 传递纹理坐标mGLTextureBuffer.position(0);GLES20.glVertexAttribPointer(vCoord, 2, GLES20.GL_FLOAT, false, 0, mGLTextureBuffer);GLES20.glEnableVertexAttribArray(vCoord);//绑定纹理GLES20.glActiveTexture(GLES20.GL_TEXTURE0);GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);// 解绑纹理GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);}&＃64;Overridepublic void onFrameAvailable(SurfaceTexture surfaceTexture) {requestRender();}
}

编写顶点着色器camera_vertex.glsl:


attribute vec4 vPosition;
attribute vec4 vCoord;
varying vec2 aCoord;uniform mat4 textureMatrix;void main(){gl_Position &＃61; vPosition;aCoord &＃61; (textureMatrix * vCoord).xy;
}


编写片段着色器camera_frag.glsl:


#extension GL_OES_EGL_image_external : require
//SurfaceTexture比较特殊
//float数据是什么精度的
precision mediump float;//采样点的坐标
varying vec2 aCoord;//采样器
uniform samplerExternalOES vTexture;void main(){//变量 接收像素值// texture2D&＃xff1a;采样器 采集 aCoord的像素//赋值给 gl_FragColor 就可以了gl_FragColor &＃61; texture2D(vTexture,aCoord);
}

加载及编译着色器程序OpenGLUtils.java:

public static String readRawTextFile(Context context, int rawId) {InputStream is &＃61; context.getResources().openRawResource(rawId);BufferedReader br &＃61; new BufferedReader(new InputStreamReader(is));String line;StringBuilder sb &＃61; new StringBuilder();try {while ((line &＃61; br.readLine()) !&＃61; null) {sb.append(line);sb.append("\n");}} catch (Exception e) {e.printStackTrace();}try {br.close();} catch (IOException e) {e.printStackTrace();}return sb.toString();}/*** 价值着色器并编译成GPU程序* &＃64;param vSource* &＃64;param fSource* &＃64;return*/public static int loadProgram(String vSource, String fSource){/*** 顶点着色器*/int vShader &＃61; GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);//加载着色器代码GLES20.glShaderSource(vShader,vSource);//编译&＃xff08;配置&＃xff09;GLES20.glCompileShader(vShader);//查看配置 是否成功int[] status &＃61; new int[1];GLES20.glGetShaderiv(vShader, GLES20.GL_COMPILE_STATUS,status,0);if(status[0] !&＃61; GLES20.GL_TRUE){//失败throw new IllegalStateException("load vertex shader:"&＃43; GLES20.glGetShaderInfoLog(vShader));}/*** 片元着色器* 流程和上面一样*/int fShader &＃61; GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);//加载着色器代码GLES20.glShaderSource(fShader,fSource);//编译&＃xff08;配置&＃xff09;GLES20.glCompileShader(fShader);//查看配置 是否成功GLES20.glGetShaderiv(fShader, GLES20.GL_COMPILE_STATUS,status,0);if(status[0] !&＃61; GLES20.GL_TRUE){//失败throw new IllegalStateException("load fragment shader:"&＃43; GLES20.glGetShaderInfoLog(vShader));}/*** 创建着色器程序*/int program &＃61; GLES20.glCreateProgram();//绑定顶点和片元GLES20.glAttachShader(program,vShader);GLES20.glAttachShader(program,fShader);//链接着色器程序GLES20.glLinkProgram(program);//获得状态GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS,status,0);if(status[0] !&＃61; GLES20.GL_TRUE){throw new IllegalStateException("link program:"&＃43; GLES20.glGetProgramInfoLog(program));}GLES20.glDeleteShader(vShader);GLES20.glDeleteShader(fShader);return program;}

结合CameraX用起来MainActivity.java:

public class MainActivity extends AppCompatActivity {private GLCameraView camera_preview;static {System.loadLibrary("native-lib");}&＃64;Overrideprotected void onCreate(Bundle savedInstanceState) {super.onCreate(savedInstanceState);setContentView(R.layout.activity_main);camera_preview &＃61; findViewById(R.id.camera_preview);if (allPermissionsGranted()) {startCamera();} else {ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, 100);}}&＃64;Overridepublic void onRequestPermissionsResult(int requestCode, &＃64;NonNull String[] permissions, &＃64;NonNull int[] grantResults) {super.onRequestPermissionsResult(requestCode, permissions, grantResults);if (requestCode &＃61;&＃61; 100) {if (allPermissionsGranted()) {startCamera();} else {Toast.makeText(this, "没有相机权限", Toast.LENGTH_LONG).show();}}}private boolean allPermissionsGranted() {return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) &＃61;&＃61; PackageManager.PERMISSION_GRANTED;}private void startCamera() {Executor executor &＃61; Executors.newSingleThreadExecutor();ListenableFuture processCameraProvider &＃61; ProcessCameraProvider.getInstance(this);processCameraProvider.addListener(new Runnable() {&＃64;Overridepublic void run() {try {ProcessCameraProvider cameraProvider &＃61; processCameraProvider.get();Preview preview &＃61; new Preview.Builder().build();camera_preview.attachPreview(preview);cameraProvider.unbindAll();cameraProvider.bindToLifecycle(MainActivity.this, CameraSelector.DEFAULT_BACK_CAMERA,preview);} catch (ExecutionException e) {e.printStackTrace();} catch (InterruptedException e) {e.printStackTrace();}}}, ContextCompat.getMainExecutor(this));}/*** A native method that is implemented by the &＃39;native-lib&＃39; native library,* which is packaged with this application.*/public native String stringFromJNI();
}

关键代码点加了点注释&＃xff0c;打完收工。

1、目前的预览竖屏看起来挺正常的&＃xff0c;但是横屏的时候预览界面明显发生变形了&＃xff0c;这个问题怎么解决呢&＃xff1f;有兴趣的童鞋可以了解下OpenGL的矩阵变换的相关知识&＃xff0c;利用矩阵变换来解决这个问题。

2、预览使用的默认的比较低的分辨率&＃xff0c;如果需要预览高分辨率需要怎么修改呢&＃xff1f;

3、笔者在预览的时候测试了一下帧率&＃xff0c;大概是每秒26帧作用&＃xff0c;如果要做到预览每秒60帧又要怎么改呢&＃xff1f;

4、入门OpenGL的童鞋应该知道VBO、 VAO 、FBO等相关概念&＃xff0c;想进一步深入学习的童鞋也可以将VBO、 VAO 、FBO与CameraX结合起来做一个实践。

CameraX虽然已经提出了两年多了&＃xff0c;但是一直还没有发布正式版&＃xff0c;貌似最近发布了一个beat版本&＃xff0c;而且笔者在学习的过程中发现相关的api也一直在变化。
所以笔者觉得CameraX是未来&＃xff0c;但不是现在。

虽然说CameraX还不稳定&＃xff0c;甚至可能还存在着各种各样的问题&＃xff0c;但是机会更加青睐的是那些未雨绸缪的人&＃xff0c;持续关注学习CameraX的演进&＃xff0c;本身就像跟着谷歌工程师学习的一个过程。

参考资料&＃xff1a;《谷歌官方》

关注我&＃xff0c;一起进步&＃xff0c;人生不止coding&＃xff01;&＃xff01;&＃xff01;