Add offload support. (#2165)

* Add offload support.

Fixes #2154.

Author: flamme

apps/OboeTester/app/src/main/cpp/FormatConverterBox.cpp

@@ -43,6 +43,14 @@ FormatConverterBox::FormatConverterBox(int32_t maxSamples,
         case oboe::AudioFormat::Unspecified:
             mSource = std::make_unique<oboe::flowgraph::SourceFloat>(1);
             break;
+        case oboe::AudioFormat::MP3:
+        case oboe::AudioFormat::AAC_LC:
+        case oboe::AudioFormat::AAC_HE_V1:
+        case oboe::AudioFormat::AAC_HE_V2:
+        case oboe::AudioFormat::AAC_ELD:
+        case oboe::AudioFormat::AAC_XHE:
+        case oboe::AudioFormat::OPUS:
+            break;
     }
 
     mSink.reset();
@@ -62,6 +70,14 @@ FormatConverterBox::FormatConverterBox(int32_t maxSamples,
         case oboe::AudioFormat::Unspecified:
             mSink = std::make_unique<oboe::flowgraph::SinkFloat>(1);
             break;
+        case oboe::AudioFormat::MP3:
+        case oboe::AudioFormat::AAC_LC:
+        case oboe::AudioFormat::AAC_HE_V1:
+        case oboe::AudioFormat::AAC_HE_V2:
+        case oboe::AudioFormat::AAC_ELD:
+        case oboe::AudioFormat::AAC_XHE:
+        case oboe::AudioFormat::OPUS:
+            break;
     }
 
     if (mSource && mSink) {

apps/OboeTester/app/src/main/java/com/mobileer/oboetester/StreamConfiguration.java

@@ -54,6 +54,7 @@ public class StreamConfiguration {
     public static final int PERFORMANCE_MODE_NONE = 10; // must match AAUDIO
     public static final int PERFORMANCE_MODE_POWER_SAVING = 11; // must match AAUDIO
     public static final int PERFORMANCE_MODE_LOW_LATENCY = 12; // must match AAUDIO
+    public static final int PERFORMANCE_MODE_POWER_SAVING_OFFLOAD = 13; // must match AAUDIO
 
     public static final int RATE_CONVERSION_QUALITY_NONE = 0; // must match Oboe
     public static final int RATE_CONVERSION_QUALITY_FASTEST = 1; // must match Oboe
@@ -406,6 +407,8 @@ static String convertPerformanceModeToText(int performanceMode) {
                 return "PS";
             case PERFORMANCE_MODE_LOW_LATENCY:
                 return "LL";
+            case PERFORMANCE_MODE_POWER_SAVING_OFFLOAD:
+                return "PSO";
             default:
                 return "??";
         }

apps/OboeTester/app/src/main/res/values/strings.xml

@@ -191,6 +191,7 @@
         <item>NONE</item>
         <item>POWER_SAVING</item>
         <item>LOW_LATENCY</item>
+        <item>POWER_SAVING_OFFLOAD</item>
     </string-array>
 
     <!--Must match SignalType in NativeAudioContext.h-->

include/oboe/AudioStream.h

@@ -572,6 +572,22 @@ class AudioStream : public AudioStreamBase {
         return oboe::Result::ErrorUnimplemented;
     }
 
+    virtual oboe::Result setOffloadDelayPadding(int32_t delayInFrames, int32_t paddingInFrames) {
+        return Result::ErrorUnimplemented;
+    }
+
+    virtual ResultWithValue<int32_t> getOffloadDelay() {
+        return ResultWithValue<int32_t>(Result::ErrorUnimplemented);
+    }
+
+    virtual ResultWithValue<int32_t> getOffloadPadding() {
+        return ResultWithValue<int32_t>(Result::ErrorUnimplemented);
+    }
+
+    virtual oboe::Result setOffloadEndOfStream() {
+        return Result::ErrorUnimplemented;
+    }
+
 protected:
 
     /**

include/oboe/AudioStreamBase.h

@@ -121,6 +121,14 @@ class AudioStreamBase {
         return mErrorCallback;
     }
 
+    /**
+     * For internal use only.
+     * @return the presentation callback object for this stream, if set.
+     */
+    std::shared_ptr<AudioStreamPresentationCallback> getPresentationCallback() const {
+        return mSharedPresentationCallback;
+    }
+
     /**
      * @return true if a data callback was set for this stream
      */
@@ -137,6 +145,13 @@ class AudioStreamBase {
         return mErrorCallback != nullptr;
     }
 
+    /**
+     * @return true if a presentation callback was set for this stream
+     */
+    bool isPresentationCallbackSpecified() const {
+        return mSharedPresentationCallback != nullptr;
+    }
+
     /**
      * @return the usage for this stream.
      */
@@ -244,6 +259,8 @@ class AudioStreamBase {
     AudioStreamErrorCallback       *mErrorCallback = nullptr;
     std::shared_ptr<AudioStreamErrorCallback> mSharedErrorCallback;
 
+    std::shared_ptr<AudioStreamPresentationCallback> mSharedPresentationCallback;
+
     /** Number of audio frames which will be requested in each callback */
     int32_t                         mFramesPerCallback = kUnspecified;
     /** Stream channel count */

include/oboe/AudioStreamBuilder.h

@@ -505,6 +505,30 @@ class AudioStreamBuilder : public AudioStreamBase {
         return this;
     }
 
+    /**
+     * Specifies an object to handle data presentation related callbacks from the underlying API.
+     * This can occur when all data queued in the audio system for an offload stream has been
+     * played.
+     *
+     * Note that presentation callbacks will only be called when a data callback has been specified
+     * and the stream is started.
+     *
+     * <strong>Important: See AudioStreamCallback for restrictions on what may be called
+     * from the callback methods.</strong>
+     *
+     * We pass a shared_ptr so that the presentationCallback object cannot be deleted before the
+     * stream is deleted. If the stream was created using a shared_ptr then the stream cannot be
+     * deleted before the presentation callback has finished running.
+     *
+     * @param sharedPresentationCallback
+     * @return pointer to the builder so calls can be chained
+     */
+    AudioStreamBuilder *setPresentationCallback(
+            std::shared_ptr<AudioStreamPresentationCallback> sharedPresentationCallback) {
+        mSharedPresentationCallback = sharedPresentationCallback;
+        return this;
+    }
+
     /**
      * Specifies an object to handle data or error related callbacks from the underlying API.
      *

include/oboe/AudioStreamCallback.h

@@ -169,6 +169,27 @@ class AudioStreamErrorCallback {
 
 };
 
+/**
+ * AudioStreamPresentationCallback defines a callback interface for
+ * being notified when a data presentation event is filed.
+ *
+ * It is used with AudioStreamBuilder::setPresentationCallback().
+ */
+class AudioStreamPresentationCallback {
+public:
+    virtual ~AudioStreamPresentationCallback() = default;
+
+    /**
+     * This will be called when all the buffers of an offloaded
+     * stream that were queued in the audio system (e.g. the
+     * combination of the Android audio framework and the device's
+     * audio hardware) have been played.
+     *
+     * @param audioStream pointer to the associated stream
+     */
+    virtual void onPresentationEnded(AudioStream* /* audioStream */) {}
+};
+
 /**
  * AudioStreamCallback defines a callback interface for:
  *

include/oboe/Definitions.h

@@ -152,6 +152,41 @@ namespace oboe {
         * Available since API 34 (U).
         */
         IEC61937 = 5, // AAUDIO_FORMAT_IEC61937
+
+        /**
+         * This format is used for audio compressed in MP3 format.
+         */
+        MP3 = 6, // AAUDIO_FORMAT_MP3
+
+        /**
+         * This format is used for audio compressed in AAC LC format.
+         */
+        AAC_LC, // AAUDIO_FORMAT_AAC_LC
+
+        /**
+         * This format is used for audio compressed in AAC HE V1 format.
+         */
+        AAC_HE_V1, // AAUDIO_FORMAT_AAC_HE_V1,
+
+        /**
+         * This format is used for audio compressed in AAC HE V2 format.
+         */
+        AAC_HE_V2, // AAUDIO_FORMAT_AAC_HE_V2
+
+        /**
+         * This format is used for audio compressed in AAC ELD format.
+         */
+        AAC_ELD, // AAUDIO_FORMAT_AAC_ELD
+
+        /**
+         * This format is used for audio compressed in AAC XHE format.
+         */
+        AAC_XHE, // AAUDIO_FORMAT_AAC_XHE
+
+        /**
+         * This format is used for audio compressed in OPUS.
+         */
+        OPUS, // AAUDIO_FORMAT_OPUS
     };
 
     /**
@@ -246,6 +281,16 @@ namespace oboe {
          * Reducing latency is most important.
          */
         LowLatency = 12, // AAUDIO_PERFORMANCE_MODE_LOW_LATENCY
+
+        /**
+         * Extending battery life is more important than low latency.
+         *
+         * This mode is not supported in input streams.
+         * This mode will play through the offloaded audio path to save battery life.
+         * With the offload playback, the default data callback size will be large and it
+         * allows data feeding thread to sleep longer time after sending enough data.
+         */
+        POWER_SAVING_OFFLOADED = 13, // AAUDIO_PERFORMANCE_MODE_POWER_SAVING_OFFLOADED
     };
 
     /**

src/aaudio/AAudioLoader.cpp

@@ -102,6 +102,10 @@ int AAudioLoader::open() {
         builder_setSpatializationBehavior = load_V_PBI("AAudioStreamBuilder_setSpatializationBehavior");
     }
 
+    if (getSdkVersion() >= __ANDROID_API_B__) {
+        builder_setPresentationEndCallback = load_V_PBPRPV("AAudioStreamBuilder_setPresentationEndCallback");
+    }
+
     builder_delete             = load_I_PB("AAudioStreamBuilder_delete");
 
 
@@ -183,6 +187,11 @@ int AAudioLoader::open() {
         aaudio_setMMapPolicy = load_I_I("AAudio_setMMapPolicy");
         aaudio_getMMapPolicy = load_I("AAudio_getMMapPolicy");
         stream_isMMapUsed = load_O_PS("AAudioStream_isMMapUsed");
+
+        stream_setOffloadDelayPadding = load_I_PSII("AAudioStream_setOffloadDelayPadding");
+        stream_getOffloadDelay = load_I_PS("AAudioStream_getOffloadDelay");
+        stream_getOffloadPadding = load_I_PS("AAudioStream_getOffloadPadding");
+        stream_setOffloadEndOfStream = load_I_PS("AAudioStream_setOffloadEndOfStream");
     }
 
     return 0;
@@ -332,6 +341,18 @@ AAudioLoader::signature_I AAudioLoader::load_I(const char *functionName) {
     return reinterpret_cast<signature_I>(proc);
 }
 
+AAudioLoader::signature_V_PBPRPV AAudioLoader::load_V_PBPRPV(const char *functionName) {
+    void *proc = dlsym(mLibHandle, functionName);
+    AAudioLoader_check(proc, functionName);
+    return reinterpret_cast<signature_V_PBPRPV>(proc);
+}
+
+AAudioLoader::signature_I_PSII AAudioLoader::load_I_PSII(const char *functionName) {
+    void *proc = dlsym(mLibHandle, functionName);
+    AAudioLoader_check(proc, functionName);
+    return reinterpret_cast<signature_I_PSII>(proc);
+}
+
 // Ensure that all AAudio primitive data types are int32_t
 #define ASSERT_INT32(type) static_assert(std::is_same<int32_t, type>::value, \
 #type" must be int32_t")

src/aaudio/AAudioLoader.h

@@ -72,6 +72,13 @@ typedef uint32_t aaudio_channel_mask_t;
 typedef int32_t aaudio_spatialization_behavior_t;
 #endif
 
+#if __NDK_MAJOR__ < 36
+// Defined in W
+typedef void (*AAudioStream_presentationEndCallback)(
+        AAudioStream* stream,
+        void* userData);
+#endif
+
 #ifndef __ANDROID_API_Q__
 #define __ANDROID_API_Q__ 29
 #endif
@@ -124,6 +131,7 @@ class AAudioLoader {
     // H = cHar
     // U = uint32_t
     // O = bOol
+    // R = pResentation end callback
 
     typedef int32_t  (*signature_I_PPB)(AAudioStreamBuilder **builder);
 
@@ -157,6 +165,10 @@ class AAudioLoader {
                                           AAudioStream_errorCallback,
                                           void *);
 
+    typedef void (*signature_V_PBPRPV)(AAudioStreamBuilder *,
+                                       AAudioStream_presentationEndCallback,
+                                       void *);
+
     typedef aaudio_format_t (*signature_F_PS)(AAudioStream *stream);
 
     typedef int32_t (*signature_I_PSPVIL)(AAudioStream *, void *, int32_t, int64_t);
@@ -176,6 +188,7 @@ class AAudioLoader {
     typedef int32_t (*signature_I_II)(int32_t, int32_t);
     typedef int32_t (*signature_I_I)(int32_t);
     typedef int32_t (*signature_I)();
+    typedef int32_t (*signature_I_PSII)(AAudioStream *, int32_t, int32_t);
 
     static AAudioLoader* getInstance(); // singleton
 
@@ -224,6 +237,7 @@ class AAudioLoader {
 
     signature_V_PBPDPV  builder_setDataCallback = nullptr;
     signature_V_PBPEPV  builder_setErrorCallback = nullptr;
+    signature_V_PBPRPV  builder_setPresentationEndCallback = nullptr;
 
     signature_I_PB      builder_delete = nullptr;
 
@@ -279,12 +293,18 @@ class AAudioLoader {
     signature_I_PS   stream_getHardwareSampleRate = nullptr;
     signature_F_PS   stream_getHardwareFormat = nullptr;
 
+
     signature_I_II   aaudio_getPlatformMMapPolicy = nullptr;
     signature_I_II   aaudio_getPlatformMMapExclusivePolicy = nullptr;
     signature_I_I    aaudio_setMMapPolicy = nullptr;
     signature_I      aaudio_getMMapPolicy = nullptr;
     signature_O_PS   stream_isMMapUsed = nullptr;
 
+    signature_I_PSII stream_setOffloadDelayPadding = nullptr;
+    signature_I_PS   stream_getOffloadDelay = nullptr;
+    signature_I_PS   stream_getOffloadPadding = nullptr;
+    signature_I_PS   stream_setOffloadEndOfStream = nullptr;
+
   private:
     AAudioLoader() {}
     ~AAudioLoader();
@@ -313,6 +333,8 @@ class AAudioLoader {
     signature_I_II      load_I_II(const char *name);
     signature_I_I       load_I_I(const char *name);
     signature_I         load_I(const char *name);
+    signature_V_PBPRPV  load_V_PBPRPV(const char *name);
+    signature_I_PSII    load_I_PSII(const char *name);
 
     void *mLibHandle = nullptr;
 };