Add personal video calls

app/jni/tgvoip/tgvoip.cpp

@@ -34,6 +34,7 @@
 #include <tgcalls/v2/InstanceV2ReferenceImpl.h>
 
 #include <tgcalls/VideoCaptureInterface.h>
+#include <tgcalls/StaticThreads.h>
 #include <platform/android/AndroidInterface.h>
 #include <platform/android/AndroidContext.h>
 
@@ -405,6 +406,11 @@ class JniWrapper {
 struct TgCallsContext {
   std::unique_ptr<tgcalls::Instance> tgcalls;
   std::shared_ptr<JniWrapper> javaController;
+  std::shared_ptr<tgcalls::VideoCaptureInterface> videoCapture;
+  std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> localVideoSink;
+  std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> remoteVideoSink;
+  bool videoEnabled = false;
+  bool frontCamera = true;
 };
 
 jbyteArray toJavaByteArray (JNIEnv *env, const std::vector<uint8_t> &data) {
@@ -464,6 +470,7 @@ JNI_OBJECT_FUNC(jlong, voip_TgCallsController, newInstance,
   env->ReleaseByteArrayElements(jEncryptionKey, (jbyte *) jEncryptionKeyData, JNI_ABORT);
 
   bool isOutgoingCall = configuration.getBoolean("isOutgoing") == JNI_TRUE;
+  bool isVideoCall = configuration.getBoolean("isVideo") == JNI_TRUE;
 
   // tgcalls::Endpoint
 
@@ -579,6 +586,16 @@ JNI_OBJECT_FUNC(jlong, voip_TgCallsController, newInstance,
 
   std::shared_ptr<JniWrapper> javaController = std::make_shared<JniWrapper>(env, thiz, tgcalls::javaTgCallsController);
 
+  auto videoPlatformContext = std::make_shared<tgcalls::AndroidContext>(env);
+  std::shared_ptr<tgcalls::VideoCaptureInterface> videoCapture =
+    tgcalls::VideoCaptureInterface::Create(
+      tgcalls::StaticThreads::getThreads(),
+      "front",
+      false,
+      videoPlatformContext
+    );
+  videoCapture->setState(isVideoCall ? tgcalls::VideoState::Active : tgcalls::VideoState::Inactive);
+
   tgcalls::Descriptor descriptor = {
     .version = version,
     .config = tgcalls::Config {
@@ -612,7 +629,7 @@ JNI_OBJECT_FUNC(jlong, voip_TgCallsController, newInstance,
       std::move(encryptionKey),
       isOutgoingCall
     ),
-    .videoCapture = nullptr,
+    .videoCapture = isVideoCall ? videoCapture : nullptr,
     .stateUpdated = [javaController](tgcalls::State state) {
       javaController->runSafely([javaController, state](JNIEnv *env) {
         jint javaState = toJavaCallState(env, state);
@@ -667,6 +684,8 @@ JNI_OBJECT_FUNC(jlong, voip_TgCallsController, newInstance,
 
   auto *context = new TgCallsContext;
   context->javaController = javaController;
+  context->videoCapture = videoCapture;
+  context->videoEnabled = isVideoCall;
   context->tgcalls = tgcalls::Meta::Create(version, std::move(descriptor));
   context->tgcalls->setNetworkType(networkType);
   context->tgcalls->setAudioOutputGainControlEnabled(audioOutputGainControlEnabled);
@@ -749,6 +768,80 @@ JNI_OBJECT_FUNC(void, voip_TgCallsController, updateAudioOutputGainControlEnable
   }
 }
 
+JNI_OBJECT_FUNC(jboolean, voip_TgCallsController, nativeSupportsVideo, jlong ptr) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  return context != nullptr && context->tgcalls != nullptr && context->tgcalls->supportsVideo()
+    ? JNI_TRUE
+    : JNI_FALSE;
+}
+
+JNI_OBJECT_FUNC(void, voip_TgCallsController, nativeSetVideoEnabled, jlong ptr, jboolean jEnabled) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  if (context == nullptr || context->tgcalls == nullptr || context->videoCapture == nullptr) {
+    return;
+  }
+  bool enabled = jEnabled == JNI_TRUE;
+  if (context->videoEnabled == enabled) {
+    return;
+  }
+  context->videoEnabled = enabled;
+  if (enabled) {
+    context->videoCapture->setState(tgcalls::VideoState::Active);
+    context->tgcalls->setVideoCapture(context->videoCapture);
+  } else {
+    context->tgcalls->setVideoCapture(nullptr);
+    context->videoCapture->setState(tgcalls::VideoState::Inactive);
+  }
+}
+
+JNI_OBJECT_FUNC(void, voip_TgCallsController, nativeSetVideoPaused, jlong ptr, jboolean jPaused) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  if (context == nullptr || context->videoCapture == nullptr || !context->videoEnabled) {
+    return;
+  }
+  context->videoCapture->setState(
+    jPaused == JNI_TRUE ? tgcalls::VideoState::Paused : tgcalls::VideoState::Active
+  );
+}
+
+JNI_OBJECT_FUNC(void, voip_TgCallsController, nativeSwitchCamera, jlong ptr) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  if (context == nullptr || context->videoCapture == nullptr) {
+    return;
+  }
+  context->frontCamera = !context->frontCamera;
+  context->videoCapture->switchToDevice(context->frontCamera ? "front" : "back", false);
+  context->videoCapture->setState(
+    context->videoEnabled ? tgcalls::VideoState::Active : tgcalls::VideoState::Inactive
+  );
+}
+
+JNI_OBJECT_FUNC(void, voip_TgCallsController, nativeSetLocalVideoOutput, jlong ptr, jobject jSink) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  if (context == nullptr || context->videoCapture == nullptr) {
+    return;
+  }
+  context->localVideoSink = jSink != nullptr
+    ? std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>(
+        webrtc::JavaToNativeVideoSink(env, jSink)
+      )
+    : nullptr;
+  context->videoCapture->setOutput(context->localVideoSink);
+}
+
+JNI_OBJECT_FUNC(void, voip_TgCallsController, nativeSetRemoteVideoOutput, jlong ptr, jobject jSink) {
+  auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
+  if (context == nullptr || context->tgcalls == nullptr) {
+    return;
+  }
+  context->remoteVideoSink = jSink != nullptr
+    ? std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>(
+        webrtc::JavaToNativeVideoSink(env, jSink)
+      )
+    : nullptr;
+  context->tgcalls->setIncomingVideoOutput(context->remoteVideoSink);
+}
+
 JNI_OBJECT_FUNC(void, voip_TgCallsController, fetchNetworkStats, jlong ptr, jobject jOutStats) {
   auto context = jni::jlong_to_ptr<TgCallsContext *>(ptr);
   if (context != nullptr && context->tgcalls != nullptr) {
@@ -868,4 +961,4 @@ jint JNI_OnLoad (JavaVM *vm, void *reserved) {
 
   return JNI_VERSION_1_6;
 }
-}
+}

app/src/main/AndroidManifest.xml

@@ -67,6 +67,7 @@
   <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
   <uses-permission android:name="android.permission.FOREGROUND_SERVICE_PHONE_CALL" />
   <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
+  <uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
   <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
   <uses-permission android:name="android.permission.MANAGE_OWN_CALLS" />
   <uses-permission android:name="android.permission.CHANGE_CONFIGURATION"
@@ -261,7 +262,7 @@
     <service
       android:name=".service.TGCallService"
       android:exported="false"
-      android:foregroundServiceType="phoneCall|microphone|mediaPlayback" />
+      android:foregroundServiceType="phoneCall|microphone|camera|mediaPlayback" />
     <receiver android:name=".receiver.VoIPMediaButtonReceiver"
       android:exported="false">
       <intent-filter>
@@ -352,6 +353,7 @@
       android:name=".MainActivity"
       android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|layoutDirection|uiMode"
       android:launchMode="singleTask"
+      android:supportsPictureInPicture="true"
       android:windowSoftInputMode="stateHidden|adjustPan"
       android:exported="true">
       <intent-filter android:label="@string/AppName">
@@ -457,4 +459,4 @@
       android:name="com.google.android.gms.car.application"
       android:resource="@xml/automotive_app_desc"/>
   </application>
-</manifest>
+</manifest>

app/src/main/java/org/telegram/messenger/voip/VideoCameraCapturer.java

@@ -0,0 +1,212 @@
+/*
+ * This file is a part of Frogram X.
+ *
+ * It bridges Telegram's tgcalls Android video source to the WebRTC camera
+ * implementation bundled with the application.
+ */
+package org.telegram.messenger.voip;
+
+import android.content.Context;
+
+import androidx.annotation.Keep;
+
+import org.thunderdog.challegram.Log;
+import org.webrtc.Camera1Enumerator;
+import org.webrtc.Camera2Enumerator;
+import org.webrtc.CameraEnumerator;
+import org.webrtc.CameraVideoCapturer;
+import org.webrtc.CapturerObserver;
+import org.webrtc.ContextUtils;
+import org.webrtc.SurfaceTextureHelper;
+
+@Keep
+public final class VideoCameraCapturer {
+  private static final int VIDEO_STATE_INACTIVE = 0;
+  private static final int VIDEO_STATE_PAUSED = 1;
+  private static final int VIDEO_STATE_ACTIVE = 2;
+
+  private static final int CAPTURE_WIDTH = 1280;
+  private static final int CAPTURE_HEIGHT = 720;
+  private static final int CAPTURE_FPS = 30;
+
+  private long nativePtr;
+  private boolean useFrontCamera = true;
+  private boolean started;
+  private boolean destroyed;
+  private int requestedState = VIDEO_STATE_INACTIVE;
+
+  private CameraVideoCapturer capturer;
+  private SurfaceTextureHelper surfaceTextureHelper;
+
+  @Keep
+  public VideoCameraCapturer () { }
+
+  @Keep
+  private synchronized void init (long nativePtr, boolean useFrontCamera) {
+    if (destroyed) {
+      return;
+    }
+    stopAndDisposeCapturer();
+    this.nativePtr = nativePtr;
+    this.useFrontCamera = useFrontCamera;
+
+    Context context = ContextUtils.getApplicationContext();
+    if (context == null) {
+      Log.e(Log.TAG_VOIP, "Video camera initialization failed: application context is unavailable");
+      return;
+    }
+
+    CameraEnumerator enumerator = Camera2Enumerator.isSupported(context)
+      ? new Camera2Enumerator(context)
+      : new Camera1Enumerator(false);
+    String deviceName = findCamera(enumerator, useFrontCamera);
+    if (deviceName == null) {
+      Log.e(Log.TAG_VOIP, "Video camera initialization failed: no camera found");
+      return;
+    }
+
+    capturer = enumerator.createCapturer(deviceName, new CameraVideoCapturer.CameraEventsHandler() {
+      @Override
+      public void onCameraError (String errorDescription) {
+        Log.e(Log.TAG_VOIP, "Video camera error: %s", errorDescription);
+      }
+
+      @Override
+      public void onCameraDisconnected () {
+        Log.w(Log.TAG_VOIP, "Video camera disconnected");
+      }
+
+      @Override
+      public void onCameraFreezed (String errorDescription) {
+        Log.e(Log.TAG_VOIP, "Video camera frozen: %s", errorDescription);
+      }
+
+      @Override
+      public void onCameraOpening (String cameraName) {
+        Log.v(Log.TAG_VOIP, "Opening video camera: %s", cameraName);
+      }
+
+      @Override
+      public void onFirstFrameAvailable () {
+        Log.v(Log.TAG_VOIP, "First local video frame is available");
+      }
+
+      @Override
+      public void onCameraClosed () {
+        Log.v(Log.TAG_VOIP, "Video camera closed");
+      }
+    });
+    if (capturer == null) {
+      Log.e(Log.TAG_VOIP, "Video camera initialization failed: capturer is unavailable");
+      return;
+    }
+
+    surfaceTextureHelper = SurfaceTextureHelper.create("FrogramVideoCamera", null);
+    if (surfaceTextureHelper == null) {
+      Log.e(Log.TAG_VOIP, "Video camera initialization failed: EGL surface is unavailable");
+      capturer.dispose();
+      capturer = null;
+      return;
+    }
+
+    CapturerObserver observer = nativeGetJavaVideoCapturerObserver(nativePtr);
+    capturer.initialize(surfaceTextureHelper, context, observer);
+    if (requestedState == VIDEO_STATE_ACTIVE) {
+      startCapture();
+    }
+  }
+
+  @Keep
+  private synchronized void onStateChanged (long nativePtr, int state) {
+    if (destroyed || this.nativePtr != nativePtr) {
+      return;
+    }
+    requestedState = state;
+    if (state == VIDEO_STATE_ACTIVE) {
+      startCapture();
+    } else if (state == VIDEO_STATE_PAUSED || state == VIDEO_STATE_INACTIVE) {
+      stopCapture();
+    }
+  }
+
+  @Keep
+  private synchronized void onAspectRatioRequested (float aspectRatio) {
+    if (capturer == null || !started || aspectRatio <= 0f) {
+      return;
+    }
+    if (aspectRatio < 1f) {
+      capturer.changeCaptureFormat(CAPTURE_HEIGHT, CAPTURE_WIDTH, CAPTURE_FPS);
+    } else {
+      capturer.changeCaptureFormat(CAPTURE_WIDTH, CAPTURE_HEIGHT, CAPTURE_FPS);
+    }
+  }
+
+  @Keep
+  private synchronized void onDestroy () {
+    destroyed = true;
+    requestedState = VIDEO_STATE_INACTIVE;
+    stopAndDisposeCapturer();
+    nativePtr = 0;
+  }
+
+  private void startCapture () {
+    if (capturer == null || started) {
+      return;
+    }
+    try {
+      capturer.startCapture(CAPTURE_WIDTH, CAPTURE_HEIGHT, CAPTURE_FPS);
+      started = true;
+    } catch (Throwable t) {
+      Log.e(Log.TAG_VOIP, "Unable to start video capture", t);
+    }
+  }
+
+  private void stopCapture () {
+    if (capturer == null || !started) {
+      return;
+    }
+    try {
+      capturer.stopCapture();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+    } catch (Throwable t) {
+      Log.e(Log.TAG_VOIP, "Unable to stop video capture", t);
+    }
+    started = false;
+  }
+
+  private void stopAndDisposeCapturer () {
+    stopCapture();
+    if (capturer != null) {
+      try {
+        capturer.dispose();
+      } catch (Throwable t) {
+        Log.e(Log.TAG_VOIP, "Unable to dispose video capturer", t);
+      }
+      capturer = null;
+    }
+    if (surfaceTextureHelper != null) {
+      try {
+        surfaceTextureHelper.dispose();
+      } catch (Throwable t) {
+        Log.e(Log.TAG_VOIP, "Unable to dispose video surface", t);
+      }
+      surfaceTextureHelper = null;
+    }
+  }
+
+  private static String findCamera (CameraEnumerator enumerator, boolean front) {
+    String fallback = null;
+    for (String deviceName : enumerator.getDeviceNames()) {
+      if (fallback == null) {
+        fallback = deviceName;
+      }
+      if ((front && enumerator.isFrontFacing(deviceName)) || (!front && enumerator.isBackFacing(deviceName))) {
+        return deviceName;
+      }
+    }
+    return fallback;
+  }
+
+  private static native CapturerObserver nativeGetJavaVideoCapturerObserver (long nativePtr);
+}