diff --git a/.gitignore b/.gitignore index 398126e..521957d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .DS_Store .dart_tool/ +.vscode + .idea .packages .pub/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 532a499..fb052f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,36 @@ +## 0.7.2 + +* iOS: Reset previous audio session category on onCancel (#67) + +## 0.7.1 + +### !!! This version changes the API !!! ++ Add StreamTransformer for easier processing +* Fix Big/Litte endian issues +* Change type of get sampleRate from double to int +* Fix parameter getters potentially never returning + +## 0.7.0-dev + +### !!! This version changes the API !!! +* Change return value of `microphone(...)` from `Future?>` to `Stream` + +## 0.6.5 +* Fixed sampleRate settings to be adapted to iOS + +## 0.6.4 + +* Change interface from having const default values to taking nullable parameters (#54) +* Make default values publicly accessible + +## 0.6.3 + +* Switch to a different MacOS backend to resolve issues of white noise (#49) + +### 0.6.2 + +* Upgrade `permission_handler` to version 10.0.0 and update compileSdk to 33 accordingly + ### 0.6.1 * Fix issues of the Audio Recorder not always being properly reinitialised on android diff --git a/README.md b/README.md index 4dea2ae..632226f 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,44 @@ -# mic_stream: 0.6.1 +# mic_stream: 0.7.2 [Flutter Plugin] Provides a tool to get the microphone input as 8 or 16 bit PCM Stream. +32 bit and floating point PCM are experimental WIP. ## About mic_stream: -As Flutter still lacks some functionality, this plugin aims to provide the possibility to easily get an audio stream from the microphone, using a simple java and swift implementation. +As Flutter still lacks some functionality, this plugin aims to provide the possibility to easily get an audio stream from the microphone of mobile devices. ## How to use: -The plugin provides one method: +The plugin mainly provides one method to provide a raw audio stream: -`Future> MicStream.microphone({options})` +`Stream MicStream.microphone({options})` + +and a `StreamTransformer` to provide a Stream of individual samples (not lists of samples): + +`MicStream.toSampleStream` + +that you can use to transform your mic stream: + +`stream.transform(MicStream.toSampleStream)` Listening to this stream starts the audio recorder while cancelling the subscription stops the stream. -The plugin also provides information about some properties: +Available options are as follows: +```dart +audioSource: AudioSource // The microphone you want to record from +sampleRate: int // The amount of data points to record per second +channelConfig: ChannelConfig // Mono or Stereo +audioFormat: AudioFormat // 8 bit PCM or 16 bit PCM. Other formats are not yet supported ``` -Future sampleRate = await MicStream.sampleRate; + +Some configuration options are platform dependent and can differ from the originally configured ones. +You can check the real values using: + +```dart +Future sampleRate = await MicStream.sampleRate; Future bitDepth = await MicStream.bitDepth; Future bufferSize = await MicStream.bufferSize; ``` @@ -38,6 +57,9 @@ In the Info.plist: Microphone access required ``` +You can disable the permission request dialogue by calling +`MicStream.shouldRequestPermission(false)` +This _will_ lead to an error if no permission to record audio has been requested, though. For MacOS: @@ -54,6 +76,11 @@ Stream> stream = await MicStream.microphone(sampleRate: 44100); StreamSubscription> listener = stream.listen((samples) => print(samples)); ``` +``` +// Transform the stream and print each sample individually +stream.transform(MicStream.toSampleStream).listen(print); +``` + ``` // Cancel the subscription listener.cancel() diff --git a/android/build.gradle b/android/build.gradle index 0f8c32f..543a6d8 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -22,11 +22,13 @@ rootProject.allprojects { apply plugin: 'com.android.library' android { - compileSdkVersion 29 + namespace 'com.code.aaron.micstream' + + compileSdkVersion 33 defaultConfig { minSdkVersion 16 - targetSdkVersion 29 + targetSdkVersion 33 testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } diff --git a/android/gradle.properties b/android/gradle.properties index d2032bc..08f2b5f 100644 --- a/android/gradle.properties +++ b/android/gradle.properties @@ -1,4 +1,3 @@ org.gradle.jvmargs=-Xmx1536M android.enableJetifier=true android.useAndroidX=true -android.enableR8=true diff --git a/android/src/main/java/com/code/aaron/micstream/MicStreamPlugin.java b/android/src/main/java/com/code/aaron/micstream/MicStreamPlugin.java index daeb37e..07be9b8 100644 --- a/android/src/main/java/com/code/aaron/micstream/MicStreamPlugin.java +++ b/android/src/main/java/com/code/aaron/micstream/MicStreamPlugin.java @@ -1,16 +1,19 @@ package com.code.aaron.micstream; -import java.lang.Math; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; -import android.annotation.TargetApi; +import android.annotation.SuppressLint; import android.media.AudioFormat; import android.media.AudioRecord; import android.media.MediaRecorder; import android.os.Handler; import android.os.Looper; +import androidx.annotation.NonNull; + import io.flutter.embedding.engine.plugins.FlutterPlugin; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.EventChannel; @@ -25,7 +28,6 @@ * and the example of the streams_channel (v0.2.2) plugin */ -@TargetApi(16) // Should be unnecessary, but isn't // fix build.gradle...? public class MicStreamPlugin implements FlutterPlugin, EventChannel.StreamHandler, MethodCallHandler { private static final String MICROPHONE_CHANNEL_NAME = "aaron.code.com/mic_stream"; private static final String MICROPHONE_METHOD_CHANNEL_NAME = "aaron.code.com/mic_stream_method_channel"; @@ -38,7 +40,7 @@ public void onAttachedToEngine(FlutterPluginBinding binding) { /// Cleanup after connection loss to flutter @Override - public void onDetachedFromEngine(FlutterPluginBinding binding) { + public void onDetachedFromEngine(@NonNull FlutterPluginBinding binding) { onCancel(null); } @@ -73,10 +75,10 @@ private volatile boolean record = false; // Method channel handlers to get sample rate / bit-depth @Override - public void onMethodCall(MethodCall call, Result result) { + public void onMethodCall(MethodCall call, @NonNull Result result) { switch (call.method) { case "getSampleRate": - result.success((double)this.actualSampleRate); // cast to double just for compatibility with the iOS version + result.success((double) this.actualSampleRate); // cast to double just for compatibility with the iOS version break; case "getBitDepth": result.success(this.actualBitDepth); @@ -90,7 +92,8 @@ public void onMethodCall(MethodCall call, Result result) { } } - private void initRecorder () { + @SuppressLint("MissingPermission") + private void initRecorder() { // Try to initialize and start the recorder recorder = new AudioRecord(AUDIO_SOURCE, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, BUFFER_SIZE); if (recorder.getState() != AudioRecord.STATE_INITIALIZED) { @@ -108,23 +111,42 @@ public void run() { isRecording = true; actualSampleRate = recorder.getSampleRate(); - actualBitDepth = (recorder.getAudioFormat() == AudioFormat.ENCODING_PCM_8BIT ? 8 : 16); + switch (recorder.getAudioFormat()) { + case AudioFormat.ENCODING_PCM_8BIT: + actualBitDepth = 8; + break; + case AudioFormat.ENCODING_PCM_16BIT: + actualBitDepth = 16; + break; + case AudioFormat.ENCODING_PCM_32BIT: + actualBitDepth = 32; + break; + case AudioFormat.ENCODING_PCM_FLOAT: + actualBitDepth = 32; + break; + } // Wait until recorder is initialised while (recorder == null || recorder.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING); + // Allocate a new buffer to write data to + ByteBuffer data = ByteBuffer.allocateDirect(BUFFER_SIZE); + + // Set ByteOrder to native + ByteOrder nativeOrder = ByteOrder.nativeOrder(); + data.order(nativeOrder); + System.out.println("mic_stream: Using native byte order " + nativeOrder); + // Repeatedly push audio samples to stream while (record) { - - // Read audio data into new byte array - byte[] data = new byte[BUFFER_SIZE]; - recorder.read(data, 0, BUFFER_SIZE); + // Read audio data into buffer + recorder.read(data, BUFFER_SIZE, AudioRecord.READ_BLOCKING); // push data into stream try { - eventSink.success(data); + eventSink.success(data.array()); } catch (IllegalArgumentException e) { - System.out.println("mic_stream: " + Arrays.hashCode(data) + " is not valid!"); + System.out.println("mic_stream: " + data + " is not valid!"); eventSink.error("-1", "Invalid Data", e); } } @@ -135,8 +157,8 @@ public void run() { /// Bug fix by https://github.com/Lokhozt /// following https://github.com/flutter/flutter/issues/34993 private static class MainThreadEventSink implements EventChannel.EventSink { - private EventChannel.EventSink eventSink; - private Handler handler; + private final EventChannel.EventSink eventSink; + private final Handler handler; MainThreadEventSink(EventChannel.EventSink eventSink) { this.eventSink = eventSink; @@ -179,29 +201,18 @@ public void run() { public void onListen(Object args, final EventChannel.EventSink eventSink) { if (isRecording) return; + // Read and validate AudioRecord parameters ArrayList config = (ArrayList) args; - - // Set parameters, if available - switch(config.size()) { - case 4: - AUDIO_FORMAT = config.get(3); - case 3: - CHANNEL_CONFIG = config.get(2); - case 2: - SAMPLE_RATE = config.get(1); - case 1: - AUDIO_SOURCE = config.get(0); - default: - try { - BUFFER_SIZE = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT); - } catch (Exception e) { - eventSink.error("-3", "Invalid AudioRecord parameters", e); - } - } - - if(AUDIO_FORMAT != AudioFormat.ENCODING_PCM_8BIT && AUDIO_FORMAT != AudioFormat.ENCODING_PCM_16BIT) { - eventSink.error("-3", "Invalid Audio Format specified", null); - return; + try { + AUDIO_SOURCE = config.get(0); + SAMPLE_RATE = config.get(1); + CHANNEL_CONFIG = config.get(2); + AUDIO_FORMAT = config.get(3); + BUFFER_SIZE = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT); + } catch (java.lang.IndexOutOfBoundsException e) { + eventSink.error("-4", "Invalid number of parameteres. Expected 4, got " + config.size(), e); + } catch (Exception e) { + eventSink.error("-3", "Invalid AudioRecord parameters", e); } this.eventSink = new MainThreadEventSink(eventSink); diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index cb32f5f..6e3eb8f 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -26,7 +26,7 @@ apply plugin: 'kotlin-android' apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" android { - compileSdkVersion 31 + namespace 'com.code.aaron.micstream' sourceSets { main.java.srcDirs += 'src/main/kotlin' @@ -40,7 +40,8 @@ android { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId "com.aaron.mic_stream" minSdkVersion 16 - targetSdkVersion 30 + targetSdkVersion 33 + compileSdkVersion 33 versionCode flutterVersionCode.toInteger() versionName flutterVersionName } @@ -52,6 +53,7 @@ android { signingConfig signingConfigs.debug } } + namespace 'com.aaron.mic_stream' } flutter { diff --git a/example/android/app/src/debug/AndroidManifest.xml b/example/android/app/src/debug/AndroidManifest.xml index 0495c6c..f880684 100644 --- a/example/android/app/src/debug/AndroidManifest.xml +++ b/example/android/app/src/debug/AndroidManifest.xml @@ -1,5 +1,4 @@ - + diff --git a/example/android/app/src/main/AndroidManifest.xml b/example/android/app/src/main/AndroidManifest.xml index e4a6582..499834e 100644 --- a/example/android/app/src/main/AndroidManifest.xml +++ b/example/android/app/src/main/AndroidManifest.xml @@ -1,5 +1,4 @@ - + - - - diff --git a/example/android/app/src/main/kotlin/com/aaron/mic_stream/MainActivity.kt b/example/android/app/src/main/kotlin/com/aaron/mic_stream/MainActivity.kt deleted file mode 100644 index 59097fd..0000000 --- a/example/android/app/src/main/kotlin/com/aaron/mic_stream/MainActivity.kt +++ /dev/null @@ -1,6 +0,0 @@ -package com.aaron.mic_stream - -import io.flutter.embedding.android.FlutterActivity - -class MainActivity: FlutterActivity() { -} diff --git a/example/android/app/src/main/kotlin/com/example/example/MainActivity.kt b/example/android/app/src/main/kotlin/com/example/example/MainActivity.kt deleted file mode 100644 index e793a00..0000000 --- a/example/android/app/src/main/kotlin/com/example/example/MainActivity.kt +++ /dev/null @@ -1,6 +0,0 @@ -package com.example.example - -import io.flutter.embedding.android.FlutterActivity - -class MainActivity: FlutterActivity() { -} diff --git a/example/android/app/src/profile/AndroidManifest.xml b/example/android/app/src/profile/AndroidManifest.xml index c208884..f880684 100644 --- a/example/android/app/src/profile/AndroidManifest.xml +++ b/example/android/app/src/profile/AndroidManifest.xml @@ -1,5 +1,4 @@ - + diff --git a/example/android/build.gradle b/example/android/build.gradle index 0c8cd4f..f0f44d5 100644 --- a/example/android/build.gradle +++ b/example/android/build.gradle @@ -1,12 +1,12 @@ buildscript { - ext.kotlin_version = '1.5.10' + ext.kotlin_version = '1.6.21' repositories { google() jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:4.0.1' + classpath 'com.android.tools.build:gradle:7.4.2' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } @@ -26,6 +26,6 @@ subprojects { project.evaluationDependsOn(':app') } -task clean(type: Delete) { +tasks.register("clean", Delete) { delete rootProject.buildDir } diff --git a/example/android/gradle.properties b/example/android/gradle.properties index 38c8d45..b9a9a24 100644 --- a/example/android/gradle.properties +++ b/example/android/gradle.properties @@ -1,4 +1,6 @@ org.gradle.jvmargs=-Xmx1536M -android.enableR8=true android.useAndroidX=true android.enableJetifier=true +android.defaults.buildfeatures.buildconfig=true +android.nonTransitiveRClass=false +android.nonFinalResIds=false diff --git a/example/android/gradle/wrapper/gradle-wrapper.properties b/example/android/gradle/wrapper/gradle-wrapper.properties index e5e73da..89e56bd 100644 --- a/example/android/gradle/wrapper/gradle-wrapper.properties +++ b/example/android/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-all.zip diff --git a/example/ios/Flutter/AppFrameworkInfo.plist b/example/ios/Flutter/AppFrameworkInfo.plist index 6b4c0f7..4f8d4d2 100644 --- a/example/ios/Flutter/AppFrameworkInfo.plist +++ b/example/ios/Flutter/AppFrameworkInfo.plist @@ -21,6 +21,6 @@ CFBundleVersion 1.0 MinimumOSVersion - 8.0 + 11.0 diff --git a/example/ios/Podfile b/example/ios/Podfile index 1e8c3c9..88359b2 100644 --- a/example/ios/Podfile +++ b/example/ios/Podfile @@ -1,5 +1,5 @@ # Uncomment this line to define a global platform for your project -# platform :ios, '9.0' +# platform :ios, '11.0' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/example/ios/Runner.xcodeproj/project.pbxproj b/example/ios/Runner.xcodeproj/project.pbxproj index fdcb86b..ffd6fff 100644 --- a/example/ios/Runner.xcodeproj/project.pbxproj +++ b/example/ios/Runner.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 50; + objectVersion = 54; objects = { /* Begin PBXBuildFile section */ @@ -163,7 +163,7 @@ 97C146E61CF9000F007C117D /* Project object */ = { isa = PBXProject; attributes = { - LastUpgradeCheck = 1020; + LastUpgradeCheck = 1300; ORGANIZATIONNAME = ""; TargetAttributes = { 97C146ED1CF9000F007C117D = { @@ -207,6 +207,7 @@ /* Begin PBXShellScriptBuildPhase section */ 3B06AD1E1E4923F5004D2608 /* Thin Binary */ = { isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); @@ -238,6 +239,7 @@ }; 9740EEB61CF901F6004384FC /* Run Script */ = { isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); @@ -347,7 +349,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.0; + IPHONEOS_DEPLOYMENT_TARGET = 11.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; @@ -363,7 +365,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = TM2B4SJXNJ; + DEVELOPMENT_TEAM = ""; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -378,7 +380,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - PRODUCT_BUNDLE_IDENTIFIER = "com.aaron.mic-stream-example"; + PRODUCT_BUNDLE_IDENTIFIER = "com.example.mic-tester"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_VERSION = 5.0; @@ -433,7 +435,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.0; + IPHONEOS_DEPLOYMENT_TARGET = 11.0; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; @@ -482,7 +484,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 8.0; + IPHONEOS_DEPLOYMENT_TARGET = 11.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; @@ -500,7 +502,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = TM2B4SJXNJ; + DEVELOPMENT_TEAM = ""; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -515,7 +517,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - PRODUCT_BUNDLE_IDENTIFIER = "com.aaron.mic-stream-example"; + PRODUCT_BUNDLE_IDENTIFIER = "com.example.mic-tester"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; @@ -531,7 +533,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = TM2B4SJXNJ; + DEVELOPMENT_TEAM = ""; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -546,7 +548,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - PRODUCT_BUNDLE_IDENTIFIER = "com.aaron.mic-stream-example"; + PRODUCT_BUNDLE_IDENTIFIER = "com.example.mic-tester"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_VERSION = 5.0; diff --git a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index a28140c..c87d15a 100644 --- a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -1,6 +1,6 @@ - - - - + + - - + CADisableMinimumFrameDurationOnPhone + CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable @@ -24,6 +26,8 @@ NSMicrophoneUsageDescription Microphone access required + UIApplicationSupportsIndirectInputEvents + UILaunchStoryboardName LaunchScreen UIMainStoryboardFile diff --git a/example/lib/main.dart b/example/lib/main.dart index c9f9ad0..a79de6f 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,11 +1,9 @@ import 'dart:async'; import 'dart:math'; import 'dart:core'; +import 'dart:typed_data'; -import 'package:flutter/widgets.dart'; import 'package:flutter/material.dart'; -import 'package:flutter/animation.dart'; -import 'package:flutter/rendering.dart'; import 'package:mic_stream/mic_stream.dart'; @@ -15,7 +13,7 @@ enum Command { change, } -const AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT; +int screenWidth = 0; void main() => runApp(MicStreamExampleApp()); @@ -26,15 +24,14 @@ class MicStreamExampleApp extends StatefulWidget { class _MicStreamExampleAppState extends State with SingleTickerProviderStateMixin, WidgetsBindingObserver { - Stream? stream; + Stream? stream; late StreamSubscription listener; - List? currentSamples = []; - List visibleSamples = []; - int? localMax; - int? localMin; - - Random rng = new Random(); + List? waveSamples; + List? intensitySamples; + int sampleIndex = 0; + double localMax = 0; + double localMin = 0; // Refreshes the Widget for every possible tick to force a rebuild of the sound wave late AnimationController controller; @@ -48,12 +45,11 @@ class _MicStreamExampleAppState extends State int page = 0; List state = ["SoundWavePage", "IntensityWavePage", "InformationPage"]; - @override void initState() { print("Init application"); super.initState(); - WidgetsBinding.instance!.addObserver(this); + WidgetsBinding.instance.addObserver(this); setState(() { initPlatformState(); }); @@ -62,7 +58,7 @@ class _MicStreamExampleAppState extends State void _controlPage(int index) => setState(() => page = index); // Responsible for switching between recording / idle state - void _controlMicStream({Command command: Command.change}) async { + void _controlMicStream({Command command = Command.change}) async { switch (command) { case Command.change: _changeListening(); @@ -79,104 +75,82 @@ class _MicStreamExampleAppState extends State Future _changeListening() async => !isRecording ? await _startListening() : _stopListening(); - late int bytesPerSample; late int samplesPerSecond; Future _startListening() async { - print("START LISTENING"); if (isRecording) return false; - // if this is the first time invoking the microphone() - // method to get the stream, we don't yet have access - // to the sampleRate and bitDepth properties - print("wait for stream"); - // Default option. Set to false to disable request permission dialogue MicStream.shouldRequestPermission(true); - stream = await MicStream.microphone( + stream = MicStream.microphone( audioSource: AudioSource.DEFAULT, - sampleRate: 1000 * (rng.nextInt(50) + 30), + sampleRate: 48000, channelConfig: ChannelConfig.CHANNEL_IN_MONO, - audioFormat: AUDIO_FORMAT); - // after invoking the method for the first time, though, these will be available; - // It is not necessary to setup a listener first, the stream only needs to be returned first - print("Start Listening to the microphone, sample rate is ${await MicStream.sampleRate}, bit depth is ${await MicStream.bitDepth}, bufferSize: ${await MicStream.bufferSize}"); - bytesPerSample = (await MicStream.bitDepth)! ~/ 8; - samplesPerSecond = (await MicStream.sampleRate)!.toInt(); - localMax = null; - localMin = null; - + audioFormat: AudioFormat.ENCODING_PCM_16BIT); + listener = stream! + .transform(MicStream.toSampleStream) + .listen(_processSamples); + listener.onError(print); + print("Start listening to the microphone, sample rate is ${await MicStream.sampleRate}, bit depth is ${await MicStream.bitDepth}, bufferSize: ${await MicStream.bufferSize}"); + + localMax = 0; + localMin = 0; + + bytesPerSample = await MicStream.bitDepth ~/ 8; + samplesPerSecond = await MicStream.sampleRate; setState(() { isRecording = true; startTime = DateTime.now(); }); - visibleSamples = []; - listener = stream!.listen(_calculateSamples); return true; } - void _calculateSamples(samples) { - if (page == 0) - _calculateWaveSamples(samples); - else if (page == 1) - _calculateIntensitySamples(samples); - } + void _processSamples(_sample) async { + if (screenWidth == 0) return; - void _calculateWaveSamples(samples) { - bool first = true; - visibleSamples = []; - int tmp = 0; - for (int sample in samples) { - if (sample > 128) sample -= 255; - if (first) { - tmp = sample * 128; - } else { - tmp += sample; - visibleSamples.add(tmp); - - localMax ??= visibleSamples.last; - localMin ??= visibleSamples.last; - localMax = max(localMax!, visibleSamples.last); - localMin = min(localMin!, visibleSamples.last); - - tmp = 0; - } - first = !first; + double sample = 0; + if ("${_sample.runtimeType}" == "(int, int)" || "${_sample.runtimeType}" == "(double, double)") { + sample = 0.5 * (_sample.$1 + _sample.$2); + } else { + sample = _sample.toDouble(); } - print(visibleSamples); - } + waveSamples ??= List.filled(screenWidth, 0); - void _calculateIntensitySamples(samples) { - currentSamples ??= []; - int currentSample = 0; - eachWithIndex(samples, (i, int sample) { - currentSample += sample; - if ((i % bytesPerSample) == bytesPerSample-1) { - currentSamples!.add(currentSample); - currentSample = 0; - } - }); + final overridden = waveSamples![sampleIndex]; + waveSamples![sampleIndex] = sample; + sampleIndex = (sampleIndex + 1) % screenWidth; - if (currentSamples!.length >= samplesPerSecond/10) { - visibleSamples.add(currentSamples!.map((i) => i).toList().reduce((a, b) => a+b)); - localMax ??= visibleSamples.last; - localMin ??= visibleSamples.last; - localMax = max(localMax!, visibleSamples.last); - localMin = min(localMin!, visibleSamples.last); - currentSamples = []; - setState(() {}); + if (overridden == localMax) { + localMax = 0; + for (final val in waveSamples!) { + localMax = max(localMax, val); + } + } else if (overridden == localMin) { + localMin = 0; + for (final val in waveSamples!) { + localMin = min(localMin, val); + } + } else { + if (sample > 0) localMax = max(localMax, sample); + else localMin = min(localMin, sample); } + + _calculateIntensitySamples(); + } + + void _calculateIntensitySamples() { } bool _stopListening() { if (!isRecording) return false; - print("Stop Listening to the microphone"); + print("Stop listening to the microphone"); listener.cancel(); setState(() { isRecording = false; - currentSamples = null; + waveSamples = List.filled(screenWidth, 0); + intensitySamples = List.filled(screenWidth, 0); startTime = null; }); return true; @@ -195,8 +169,7 @@ class _MicStreamExampleAppState extends State if (isRecording) setState(() {}); }) ..addStatusListener((status) { - if (status == AnimationStatus.completed) - controller.reverse(); + if (status == AnimationStatus.completed) controller.reverse(); else if (status == AnimationStatus.dismissed) controller.forward(); }) ..forward(); @@ -243,13 +216,9 @@ class _MicStreamExampleAppState extends State ), body: (page == 0 || page == 1) ? CustomPaint( - painter: WavePainter( - samples: visibleSamples, - color: _getBgColor(), - localMax: localMax, - localMin: localMin, - context: context, - ), + painter: page == 0 + ? WavePainter(samples: waveSamples, color: _getBgColor(), index: sampleIndex, localMax: localMax, localMin: localMin, context: context,) + : IntensityPainter(samples: intensitySamples, color: _getBgColor(), index: sampleIndex, localMax: localMax, localMin: localMin, context: context,) ) : Statistics( isRecording, @@ -264,8 +233,7 @@ class _MicStreamExampleAppState extends State isActive = true; print("Resume app"); - _controlMicStream( - command: memRecordingState ? Command.start : Command.stop); + _controlMicStream(command: memRecordingState ? Command.start : Command.stop); } else if (isActive) { memRecordingState = isRecording; _controlMicStream(command: Command.stop); @@ -279,41 +247,38 @@ class _MicStreamExampleAppState extends State void dispose() { listener.cancel(); controller.dispose(); - WidgetsBinding.instance!.removeObserver(this); + WidgetsBinding.instance.removeObserver(this); super.dispose(); } } class WavePainter extends CustomPainter { - int? localMax; - int? localMin; - List? samples; + int? index; + double? localMax; + double? localMin; + List? samples; late List points; Color? color; BuildContext? context; Size? size; - // Set max val possible in stream, depending on the config - // int absMax = 255*4; //(AUDIO_FORMAT == AudioFormat.ENCODING_PCM_8BIT) ? 127 : 32767; - // int absMin; //(AUDIO_FORMAT == AudioFormat.ENCODING_PCM_8BIT) ? 127 : 32767; - - WavePainter({this.samples, this.color, this.context, this.localMax, this.localMin}); + WavePainter({this.samples, this.color, this.context, this.index, this.localMax, this.localMin}); @override void paint(Canvas canvas, Size? size) { this.size = context!.size; size = this.size; + if (size == null) return; + screenWidth = size.width.toInt(); Paint paint = new Paint() ..color = color! ..strokeWidth = 1.0 ..style = PaintingStyle.stroke; - if (samples!.length == 0) - return; - - - points = toPoints(samples); + samples ??= List.filled(screenWidth, 0); + index ??= 0; + points = toPoints(samples!, index!); Path path = new Path(); path.addPolygon(points, false); @@ -325,21 +290,49 @@ class WavePainter extends CustomPainter { bool shouldRepaint(CustomPainter oldPainting) => true; // Maps a list of ints and their indices to a list of points on a cartesian grid - List toPoints(List? samples) { + List toPoints(List samples, int index) { List points = []; - if (samples == null) - samples = List.filled(size!.width.toInt(), (0.5).toInt()); - double pixelsPerSample = size!.width/samples.length; - for (int i = 0; i < samples.length; i++) { - var point = Offset(i * pixelsPerSample, 0.5 * size!.height * pow((samples[i] - localMin!)/(localMax! - localMin!), 5)); + double totalMax = max(-1 * localMin!, localMax!); + double maxHeight = 0.5 * size!.height; + for (int i = 0; i < screenWidth; i++) { + double height = maxHeight + ((totalMax == 0 || index == 0) ? 0 : (samples[(i + index) % index] / totalMax * maxHeight)); + var point = Offset(i.toDouble(), height); points.add(point); } return points; } +} - double project(int val, int max, double height) { - double waveHeight = (max == 0) ? val.toDouble() : (val / max) * 0.5 * height; - return waveHeight + 0.5 * height; +class IntensityPainter extends CustomPainter { + int? index; + double? localMax; + double? localMin; + List? samples; + late List points; + Color? color; + BuildContext? context; + Size? size; + + IntensityPainter({this.samples, this.color, this.context, this.index, this.localMax, this.localMin}); + + @override + void paint(Canvas canvas, Size? size) { + } + + @override + bool shouldRepaint(CustomPainter oldPainting) => true; + + // Maps a list of ints and their indices to a list of points on a cartesian grid + List toPoints(List? samples) { + return points; + } + + double project(double val, double max, double height) { + if (max == 0) { + return 0.5 * height; + } + var rv = val / max * 0.5 * height; + return rv; } } @@ -370,7 +363,6 @@ class Statistics extends StatelessWidget { } } - Iterable eachWithIndex( Iterable items, E Function(int index, T item) f) { var index = 0; @@ -382,4 +374,3 @@ Iterable eachWithIndex( return items; } - diff --git a/example/macos/Flutter/GeneratedPluginRegistrant.swift b/example/macos/Flutter/GeneratedPluginRegistrant.swift index 1e25b10..1aa0366 100644 --- a/example/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/example/macos/Flutter/GeneratedPluginRegistrant.swift @@ -6,9 +6,7 @@ import FlutterMacOS import Foundation import mic_stream -import path_provider_macos func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { MicStreamPlugin.register(with: registry.registrar(forPlugin: "MicStreamPlugin")) - PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) } diff --git a/example/macos/Podfile b/example/macos/Podfile index dade8df..049abe2 100644 --- a/example/macos/Podfile +++ b/example/macos/Podfile @@ -1,4 +1,4 @@ -platform :osx, '10.11' +platform :osx, '10.14' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/example/macos/Runner.xcodeproj/project.pbxproj b/example/macos/Runner.xcodeproj/project.pbxproj index 0a5c946..a845323 100644 --- a/example/macos/Runner.xcodeproj/project.pbxproj +++ b/example/macos/Runner.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 51; + objectVersion = 54; objects = { /* Begin PBXAggregateTarget section */ @@ -203,7 +203,7 @@ isa = PBXProject; attributes = { LastSwiftUpdateCheck = 0920; - LastUpgradeCheck = 0930; + LastUpgradeCheck = 1300; ORGANIZATIONNAME = ""; TargetAttributes = { 33CC10EC2044A3C60003C045 = { @@ -256,6 +256,7 @@ /* Begin PBXShellScriptBuildPhase section */ 3399D490228B24CF009A79C7 /* ShellScript */ = { isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); @@ -404,7 +405,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.11; + MACOSX_DEPLOYMENT_TARGET = 10.14; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = macosx; SWIFT_COMPILATION_MODE = wholemodule; @@ -483,7 +484,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.11; + MACOSX_DEPLOYMENT_TARGET = 10.14; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = macosx; @@ -530,7 +531,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.11; + MACOSX_DEPLOYMENT_TARGET = 10.14; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = macosx; SWIFT_COMPILATION_MODE = wholemodule; diff --git a/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 2f4543e..3f9170c 100644 --- a/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -1,6 +1,6 @@ =2.12.0 <3.0.0' + sdk: '>=3.0.0 <=3.13.7' dependencies: flutter: diff --git a/ios/Classes/SwiftMicStreamPlugin.swift b/ios/Classes/SwiftMicStreamPlugin.swift index 4707579..f930239 100644 --- a/ios/Classes/SwiftMicStreamPlugin.swift +++ b/ios/Classes/SwiftMicStreamPlugin.swift @@ -2,7 +2,7 @@ import Flutter //import UIKit import AVFoundation import Dispatch - +import AVFAudio enum AudioFormat : Int { case ENCODING_PCM_8BIT=3, ENCODING_PCM_16BIT=2 } enum ChannelConfig : Int { case CHANNEL_IN_MONO=16 , CHANNEL_IN_STEREO=12 } enum AudioSource : Int { case DEFAULT } @@ -26,17 +26,22 @@ public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin var BUFFER_SIZE = 4096; var eventSink:FlutterEventSink?; var session : AVCaptureSession! - + var audioSession: AVAudioSession! + var oldAudioSessionCategory: AVAudioSession.Category? + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { switch call.method { case "getSampleRate": - result(self.actualSampleRate) + result(self.actualSampleRate)//call the actual sample rate break; case "getBitDepth": result(self.actualBitDepth) break; case "getBufferSize": - result(self.BUFFER_SIZE) + result(Int(self.audioSession.ioBufferDuration*self.audioSession.sampleRate))//calculate the true buffer size + break; + case "clean": + onCancel(withArguments: nil) break; default: result(FlutterMethodNotImplemented) @@ -45,43 +50,48 @@ public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin public func onCancel(withArguments arguments:Any?) -> FlutterError? { self.session?.stopRunning() + if let category = oldAudioSessionCategory { + try? audioSession.setCategory(category) + } return nil } public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { - if (isRecording) { return nil; } - let config = arguments as! [Int?]; - // Set parameters, if available - print(config); + let config = arguments as! [Int?] switch config.count { case 4: AUDIO_FORMAT = AudioFormat(rawValue:config[3]!)!; + if(AUDIO_FORMAT != AudioFormat.ENCODING_PCM_16BIT) { + events(FlutterError(code: "-3", message: "Currently only AudioFormat ENCODING_PCM_16BIT is supported", details:nil)) + return nil + } fallthrough case 3: CHANNEL_CONFIG = ChannelConfig(rawValue:config[2]!)!; if(CHANNEL_CONFIG != ChannelConfig.CHANNEL_IN_MONO) { - events(FlutterError(code: "-3", - message: "Currently only ChannelConfig CHANNEL_IN_MONO is supported", details:nil)) + events(FlutterError(code: "-3", message: "Currently only ChannelConfig CHANNEL_IN_MONO is supported", details:nil)) return nil } fallthrough case 2: SAMPLE_RATE = config[1]!; + if(SAMPLE_RATE<8000 || SAMPLE_RATE>48000) { + events(FlutterError(code: "-3", message: "iPhone only sample rates between 8000 and 48000 are supported", details:nil)) + return nil + } fallthrough case 1: AUDIO_SOURCE = AudioSource(rawValue:config[0]!)!; if(AUDIO_SOURCE != AudioSource.DEFAULT) { - events(FlutterError(code: "-3", - message: "Currently only default AUDIO_SOURCE (id: 0) is supported", details:nil)) + events(FlutterError(code: "-3", message: "Currently only default AUDIO_SOURCE (id: 0) is supported", details:nil)) return nil } default: - events(FlutterError(code: "-3", - message: "At least one argument (AudioSource) must be provided ", details:nil)) + events(FlutterError(code: "-3", message: "At least one argument (AudioSource) must be provided ", details:nil)) return nil } self.eventSink = events; @@ -94,24 +104,52 @@ public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin if let audioCaptureDevice : AVCaptureDevice = AVCaptureDevice.default(for:AVMediaType.audio) { self.session = AVCaptureSession() + self.audioSession = AVAudioSession.sharedInstance() do { + //magic word + //This will allow developers to specify sample rates, etc. + try session.automaticallyConfiguresApplicationAudioSession = false + try audioCaptureDevice.lockForConfiguration() + + oldAudioSessionCategory = audioSession.category + + try audioSession.setCategory(AVAudioSession.Category.record,mode: .measurement) + + try audioSession.setPreferredSampleRate(Double(SAMPLE_RATE)) + + //Calculate the time required for BufferSize + let preferredIOBufferDuration: TimeInterval = 1.0 / audioSession.sampleRate * Double(self.BUFFER_SIZE) + try audioSession.setPreferredIOBufferDuration(Double(preferredIOBufferDuration)) + + //it does not seem like this is working + //let numChannels = CHANNEL_CONFIG == ChannelConfig.CHANNEL_IN_MONO ? 1 : 2 + //try audioSession.setPreferredInputNumberOfChannels(1) + + + // print("this is the session sample rate: \(audioSession.sampleRate)") + // print("this is the session preferred sample rate: \(audioSession.preferredSampleRate)") + // print("this is the session preferred IOBufferDuration: \(audioSession.preferredIOBufferDuration)") + // print("this is the session IOBufferDuration: \(audioSession.ioBufferDuration)") + // print("this is the session preferred input number of channels: \(audioSession.preferredInputNumberOfChannels)") + // print("this is the session input number of channels: \(audioSession.inputNumberOfChannels)") + + try audioSession.setActive(true) + let audioInput = try AVCaptureDeviceInput(device: audioCaptureDevice) + + audioCaptureDevice.unlockForConfiguration() if(self.session.canAddInput(audioInput)){ self.session.addInput(audioInput) } - - //let numChannels = CHANNEL_CONFIG == ChannelConfig.CHANNEL_IN_MONO ? 1 : 2 - // setting the preferred sample rate on AVAudioSession doesn't magically change the sample rate for our AVCaptureSession - // try AVAudioSession.sharedInstance().setPreferredSampleRate(Double(SAMPLE_RATE)) - // neither does setting AVLinearPCMBitDepthKey on audioOutput.audioSettings (unavailable on iOS) // 99% sure it's not possible to set streaming sample rate/bitrate // try AVAudioSession.sharedInstance().setPreferredOutputNumberOfChannels(numChannels) + let audioOutput = AVCaptureAudioDataOutput() audioOutput.setSampleBufferDelegate(self, queue: DispatchQueue.global()) @@ -123,8 +161,10 @@ public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin self.session.startRunning() } } catch let e { - self.eventSink!(FlutterError(code: "-3", - message: "Error encountered starting audio capture, see details for more information.", details:e)) + // print("Error encountered starting audio capture, see details for more information.") + // print(e) + + self.eventSink!(FlutterError(code: "-3", message: "Error encountered starting audio capture, see details for more information.", details:e)) } } } @@ -160,9 +200,10 @@ public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin self.actualSampleRate = asbd?.pointee.mSampleRate self.actualBitDepth = asbd?.pointee.mBitsPerChannel } - + //print(actualSampleRate) + //print(audioSession.sampleRate) let data = Data(bytesNoCopy: audioBufferList.mBuffers.mData!, count: Int(audioBufferList.mBuffers.mDataByteSize), deallocator: .none) + self.eventSink!(FlutterStandardTypedData(bytes: data)) - } } diff --git a/ios/mic_stream.podspec b/ios/mic_stream.podspec index 38840d8..e797f98 100644 --- a/ios/mic_stream.podspec +++ b/ios/mic_stream.podspec @@ -16,6 +16,6 @@ Provides a tool to get the microphone input as Byte Stream s.public_header_files = 'Classes/**/*.h' s.dependency 'Flutter' - s.ios.deployment_target = '8.0' + s.ios.deployment_target = '11.0' end diff --git a/lib/mic_stream.dart b/lib/mic_stream.dart index e304e36..7849dd8 100644 --- a/lib/mic_stream.dart +++ b/lib/mic_stream.dart @@ -1,9 +1,8 @@ import 'dart:async'; import 'dart:io'; -import 'package:permission_handler/permission_handler.dart' as handler; import 'package:flutter/services.dart'; -import 'dart:typed_data'; +import 'package:permission_handler/permission_handler.dart' as handler; // In reference to the implementation of the official sensors plugin // https://github.com/flutter/plugins/tree/master/packages/sensors @@ -25,22 +24,33 @@ enum AudioSource { /// Mono: Records using one microphone; /// Stereo: Records using two spatially distant microphones (if applicable) -enum ChannelConfig { CHANNEL_IN_MONO, CHANNEL_IN_STEREO } +enum ChannelConfig { + CHANNEL_IN_MONO, + CHANNEL_IN_STEREO, +} /// Bit depth. -/// 8-bit means each sample consists of 1 byte -/// 16-bit means each sample consists of 2 consecutive bytes, in little endian -enum AudioFormat { ENCODING_PCM_8BIT, ENCODING_PCM_16BIT } +/// 8 bit means each sample consists of 1 byte +/// 16 bit means each sample consists of 2 consecutive bytes, in little endian +/// 24 bit is currently not supported (cause nobody needs this) +/// 32 bit means each sample consists of 4 consecutive bytes, in little endian +/// float is the same as 32 bit, except it represents a floating point number +enum AudioFormat { + ENCODING_PCM_8BIT, + ENCODING_PCM_16BIT, + ENCODING_PCM_FLOAT, +//ENCODING_PCM_24BIT_PACKED, + ENCODING_PCM_32BIT +} class MicStream { static bool _requestPermission = true; - static const AudioSource _DEFAULT_AUDIO_SOURCE = AudioSource.DEFAULT; - static const ChannelConfig _DEFAULT_CHANNELS_CONFIG = + static const AudioSource DEFAULT_AUDIO_SOURCE = AudioSource.DEFAULT; + static const ChannelConfig DEFAULT_CHANNELS_CONFIG = ChannelConfig.CHANNEL_IN_MONO; - static const AudioFormat _DEFAULT_AUDIO_FORMAT = - AudioFormat.ENCODING_PCM_8BIT; - static const int _DEFAULT_SAMPLE_RATE = 16000; + static const AudioFormat DEFAULT_AUDIO_FORMAT = AudioFormat.ENCODING_PCM_8BIT; + static const int DEFAULT_SAMPLE_RATE = 16000; static const int _MIN_SAMPLE_RATE = 1; static const int _MAX_SAMPLE_RATE = 100000; @@ -50,23 +60,46 @@ class MicStream { static const MethodChannel _microphoneMethodChannel = MethodChannel('aaron.code.com/mic_stream_method_channel'); - /// The actual sample rate used for streaming. This may return zero if invoked without listening to the _microphone Stream - static Future? get sampleRate => _sampleRate; - - static Future? _sampleRate; - - /// The actual bit depth used for streaming. This may return zero if invoked without listening to the _microphone Stream first. - static Future? get bitDepth => _bitDepth; - - static Future? _bitDepth; + /// The actual sample rate used for streaming. Only completes once a stream started. + static Future get sampleRate async { + _memoisedSampleRate ??= await _microphoneFuture.then((_) { + return _microphoneMethodChannel.invokeMethod("getSampleRate") + .then((value) => (value as double).toInt()); + }); + return _memoisedSampleRate!; + } + static int? _memoisedSampleRate; - /// The amount of recorded data, per sample, in bytes - static Future? get bufferSize => _bufferSize; + /// The actual bit depth used for streaming. Only completes once a stream started. + static Future get bitDepth async { + _memoisedBitDepth = await _microphoneFuture.then((_) { + return _microphoneMethodChannel.invokeMethod("getBitDepth") + .then((value) => value as int); + }); + return _memoisedBitDepth!; + } + static int? _memoisedBitDepth; - static Future? _bufferSize; + /// The amount of recorded data, per sample, in bytes. Only completes once a stream started. + static Future get bufferSize async { + _memoisedBufferSize ??= await _microphoneFuture.then((_) { + return _microphoneMethodChannel.invokeMethod("getBufferSize") + .then((value) => value as int); + }); + return _memoisedBufferSize!; + } + static int? _memoisedBufferSize; - /// The configured microphone stream and its config + /// The configured microphone stream static Stream? _microphone; + static Completer _microphoneCompleter = new Completer(); + static Future get _microphoneFuture async { + if (!_microphoneCompleter.isCompleted) { + await _microphoneCompleter.future; + } + } + + /// The configured stream config static AudioSource? __audioSource; static int? __sampleRate; static ChannelConfig? __channelConfig; @@ -78,75 +111,172 @@ class MicStream { return true; } var micStatus = await handler.Permission.microphone.request(); - return !micStatus.isDenied; + return !micStatus.isDenied && !micStatus.isPermanentlyDenied; } /// This function initializes a connection to the native backend (if not already available). /// Returns a Uint8List stream representing the captured audio. /// IMPORTANT - on iOS, there is no guarantee that captured audio will be encoded with the requested sampleRate/bitDepth. /// You must check the sampleRate and bitDepth properties of the MicStream object *after* invoking this method (though this does not need to be before listening to the returned stream). - /// This is why this method returns a Uint8List - if you request a 16-bit encoding, you will need to check that - /// the returned stream is actually returning 16-bit data, and if so, manually cast uint8List.buffer.asUint16List() + /// This is why this method returns a Uint8List - if you request a deeper encoding, + /// you will need to manually convert the returned stream to the appropriate type, + /// e.g., for 16 bit map each element using uint8List.buffer.asUint16List(). + /// Alternatively, you can call `toSampleStream(Stream)` to transform the raw stream to a more easily usable stream. + /// /// audioSource: The device used to capture audio. The default let's the OS decide. /// sampleRate: The amount of samples per second. More samples give better quality at the cost of higher data transmission /// channelConfig: States whether audio is mono or stereo - /// audioFormat: Switch between 8- and 16-bit PCM streams + /// audioFormat: Switch between 8, 16, 32 bit, and floating point PCM streams /// - static Future?> microphone( - {AudioSource audioSource: _DEFAULT_AUDIO_SOURCE, - int sampleRate: _DEFAULT_SAMPLE_RATE, - ChannelConfig channelConfig: _DEFAULT_CHANNELS_CONFIG, - AudioFormat audioFormat: _DEFAULT_AUDIO_FORMAT}) async { + static Stream microphone( + {AudioSource? audioSource, + int? sampleRate, + ChannelConfig? channelConfig, + AudioFormat? audioFormat}) { + audioSource ??= DEFAULT_AUDIO_SOURCE; + sampleRate ??= DEFAULT_SAMPLE_RATE; + channelConfig ??= DEFAULT_CHANNELS_CONFIG; + audioFormat ??= DEFAULT_AUDIO_FORMAT; + if (sampleRate < _MIN_SAMPLE_RATE || sampleRate > _MAX_SAMPLE_RATE) - throw (RangeError.range(sampleRate, _MIN_SAMPLE_RATE, _MAX_SAMPLE_RATE)); - if (_requestPermission) if (!(await permissionStatus)) - throw (PlatformException); + return Stream.error( + RangeError.range(sampleRate, _MIN_SAMPLE_RATE, _MAX_SAMPLE_RATE)); + final permissionStatus = _requestPermission + ? Stream.fromFuture(MicStream.permissionStatus) + : Stream.value(true); + + return permissionStatus.asyncExpand((grantedPermission) { + if (!grantedPermission) { + throw Exception('Microphone permission is not granted'); + } + return _setupMicStream( + audioSource!, + sampleRate!, + channelConfig!, + audioFormat!, + ); + }); + } + + static Stream _setupMicStream( + AudioSource audioSource, + int sampleRate, + ChannelConfig channelConfig, + AudioFormat audioFormat, + ) { // If first time or configs have changed reinitialise audio recorder if (audioSource != __audioSource || sampleRate != __sampleRate || channelConfig != __channelConfig || audioFormat != __audioFormat) { - //TODO: figure out whether the old stream needs to be cancelled + + // Reset runtime values + if (_microphone != null) { + var _tmpCompleter = _microphoneCompleter; + _microphoneCompleter = new Completer(); + _tmpCompleter.complete(_microphoneCompleter.future); + } + _memoisedSampleRate = null; + _memoisedBitDepth = null; + _memoisedBufferSize = null; + + // Reset configuration + __audioSource = audioSource; + __sampleRate = sampleRate; + __channelConfig = channelConfig; + __audioFormat = audioFormat; + + // Reset audio stream _microphone = _microphoneEventChannel.receiveBroadcastStream([ audioSource.index, sampleRate, channelConfig == ChannelConfig.CHANNEL_IN_MONO ? 16 : 12, - audioFormat == AudioFormat.ENCODING_PCM_8BIT ? 3 : 2 + switch (audioFormat) { + AudioFormat.ENCODING_PCM_8BIT => 3, + AudioFormat.ENCODING_PCM_16BIT => 2, +// AudioFormat.ENCODING_PCM_24BIT_PACKED => 21, + AudioFormat.ENCODING_PCM_32BIT => 22, + AudioFormat.ENCODING_PCM_FLOAT => 4 + } ]).cast(); - __audioSource = audioSource; - __sampleRate = sampleRate; - __channelConfig = channelConfig; - __audioFormat = audioFormat; } - // sampleRate/bitDepth should be populated before any attempt to consume the stream externally. - // configure these as Completers and listen to the stream internally before returning - // these will complete only when this internal listener is called - StreamSubscription? listener; - var sampleRateCompleter = new Completer(); - var bitDepthCompleter = new Completer(); - var bufferSizeCompleter = new Completer(); - _sampleRate = sampleRateCompleter.future; - _bitDepth = bitDepthCompleter.future; - _bufferSize = bufferSizeCompleter.future; - - listener = _microphone!.listen((x) async { - await listener!.cancel(); - listener = null; - sampleRateCompleter.complete(await _microphoneMethodChannel - .invokeMethod("getSampleRate") as double?); - bitDepthCompleter.complete( - await _microphoneMethodChannel.invokeMethod("getBitDepth") as int?); - bufferSizeCompleter.complete( - await _microphoneMethodChannel.invokeMethod("getBufferSize") as int?); + // Check for errors + if (_microphone == null) { + if (!_microphoneCompleter.isCompleted) { + _microphoneCompleter.completeError(StateError); + } + return Stream.error(StateError); + } + + // Force evaluation of actual config values + _microphone!.first.then((value) { + if (!_microphoneCompleter.isCompleted) { + _microphoneCompleter.complete(); + } }); - return _microphone; + return _microphone!; + } + + /// StreamTransformer to convert a raw Stream to num streams, e.g.: + /// 8 bit PCM + mono => Stream, where each int is a *signed* byte, i.e., [-2^7; 2^7) + /// 16 bit PCM + stereo => Stream<(int, int)>, where each int is a *signed* byte, i.e., [-2^15; 2^15) + /// float bit PCM + stereo => Stream<(double, double)>, with double e [-1.0; 1.0), and 32 bit precision + static StreamTransformer get toSampleStream => + // TODO: check bitDepth here already and call different handlers for every possible combination + (__channelConfig == ChannelConfig.CHANNEL_IN_MONO) + ? new StreamTransformer.fromHandlers(handleData: _expandUint8ListMono) + : new StreamTransformer.fromHandlers(handleData: _expandUint8ListStereo); + + static void _expandUint8ListMono(Uint8List raw, EventSink sink) async { + switch (await bitDepth) { + case 8: raw.buffer.asInt8List().forEach(sink.add); break; + case 16: raw.buffer.asInt16List().forEach(sink.add); break; + case 24: sink.addError("24 bit PCM encoding is not supported"); break; + case 32: (__audioFormat == AudioFormat.ENCODING_PCM_32BIT) + ? raw.buffer.asInt32List().forEach(sink.add) + : raw.buffer.asFloat32List().forEach(sink.add); + break; + default: + sink.addError("No stream configured yet"); + } + } + static void _expandUint8ListStereo(Uint8List raw, EventSink sink) async { + switch (await bitDepth) { + case 8: _listToPairList(raw.buffer.asInt8List()).forEach(sink.add); break; + case 16: _listToPairList(raw.buffer.asInt16List()).forEach(sink.add); break; + case 24: sink.addError("24 bit PCM encoding is not supported"); break; + case 32: (__audioFormat == AudioFormat.ENCODING_PCM_32BIT) + ? _listToPairList(raw.buffer.asInt32List()).forEach(sink.add) + : _listToPairList(raw.buffer.asFloat32List()).forEach(sink.add); + break; + default: + sink.addError("No stream configured yet"); + } + } + static List<(num, num)> _listToPairList(List mono) { + List<(num, num)> stereo = List.empty(growable: true); + num? first; + for (num sample in mono) { + if (first == null) { + first = sample; + } + else { + stereo.add((first, sample)); + first = null; + } + } + return stereo; + } + + static void clean() { + _microphoneMethodChannel.invokeMethod("clean"); } /// Updates flag to determine whether to request audio recording permission. Set to false to disable dialogue, set to true (default) to request permission if necessary - static bool shouldRequestPermission(bool request_permission) { - return _requestPermission = request_permission; + static bool shouldRequestPermission(bool requestPermission) { + return _requestPermission = requestPermission; } } diff --git a/macos/Classes/MicStreamPlugin.swift b/macos/Classes/MicStreamPlugin.swift index 02e67ca..6b385c1 100644 --- a/macos/Classes/MicStreamPlugin.swift +++ b/macos/Classes/MicStreamPlugin.swift @@ -1,170 +1,115 @@ import Cocoa import FlutterMacOS -//import UIKit import AVFoundation -import Dispatch - -enum AudioFormat : Int { case ENCODING_PCM_8BIT=3, ENCODING_PCM_16BIT=2 } -enum ChannelConfig : Int { case CHANNEL_IN_MONO=16 , CHANNEL_IN_STEREO=12 } -enum AudioSource : Int { case DEFAULT } - -public class SwiftMicStreamPlugin: NSObject, FlutterStreamHandler, FlutterPlugin, AVCaptureAudioDataOutputSampleBufferDelegate { - public static func register(with registrar: FlutterPluginRegistrar) { - let channel = FlutterEventChannel(name:"aaron.code.com/mic_stream", binaryMessenger: registrar.messenger) - let methodChannel = FlutterMethodChannel(name: "aaron.code.com/mic_stream_method_channel", binaryMessenger: registrar.messenger) - let instance = SwiftMicStreamPlugin() - channel.setStreamHandler(instance); - registrar.addMethodCallDelegate(instance, channel: methodChannel) - } - let isRecording:Bool = false; - var CHANNEL_CONFIG:ChannelConfig = ChannelConfig.CHANNEL_IN_MONO; - var SAMPLE_RATE:Int = 44100; // this is the sample rate the user wants - var actualSampleRate:Float64?; // this is the actual hardware sample rate the device is using - var AUDIO_FORMAT:AudioFormat = AudioFormat.ENCODING_PCM_16BIT; // this is the encoding/bit-depth the user wants - var actualBitDepth:UInt32?; // this is the actual hardware bit-depth - var AUDIO_SOURCE:AudioSource = AudioSource.DEFAULT; - var BUFFER_SIZE = 4096; - var eventSink:FlutterEventSink?; - var session : AVCaptureSession! - - public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - switch call.method { - case "getSampleRate": - result(self.actualSampleRate) - break; - case "getBitDepth": - result(self.actualBitDepth) - break; - case "getBufferSize": - result(self.BUFFER_SIZE) - break; - default: - result(FlutterMethodNotImplemented) - } +/// Notes: +/// 1. currently the only config supported is: +/// audioSource == DEFAULT +/// sampleRate == 48000 +/// channelConfig == MONO +/// audioFormat == 16BIT +/// 2. AVAudioEngine is used to acquire the audio. The previous version uses +/// AVCaptureAudioDataOutputSampleBufferDelegate, which records noise on +/// my machine +/// 3. The native audio sample is of float32 type. the samples are casted into +/// int16 to conform with the library definition + + +public class SwiftMicStreamPlugin: NSObject, FlutterPlugin, FlutterStreamHandler { + public static func register(with registrar: FlutterPluginRegistrar) { + let instance = SwiftMicStreamPlugin() + + let micChannel = FlutterEventChannel(name:"aaron.code.com/mic_stream", binaryMessenger: registrar.messenger) + micChannel.setStreamHandler(instance); + + let channel = FlutterMethodChannel(name: "aaron.code.com/mic_stream_method_channel", binaryMessenger: registrar.messenger) + registrar.addMethodCallDelegate(instance, channel: channel) + } + + var sampleRate: Float64? = 48000; + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "getSampleRate": + result(self.sampleRate) + break; + case "getBitDepth": + result(16) // always 16 + break; + case "getBufferSize": + result(-1) // not given, check received buffer length instead + default: + result(FlutterMethodNotImplemented) } - - public func onCancel(withArguments arguments:Any?) -> FlutterError? { - self.session?.stopRunning() - return nil + } + + var audioEngine = AVAudioEngine(); + var isRecording = false; + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + if (isRecording) { + NSLog("onListen being called while recording") + return FlutterError() } + isRecording = true; - public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { - NSLog("ON LISTEN CALLED................... *"); - if (isRecording) { - return nil; - } - - let config = arguments as! [Int?]; - // Set parameters, if available - print(config); - switch config.count { - case 4: - AUDIO_FORMAT = AudioFormat(rawValue:config[3]!)!; - fallthrough - case 3: - CHANNEL_CONFIG = ChannelConfig(rawValue:config[2]!)!; - if(CHANNEL_CONFIG != ChannelConfig.CHANNEL_IN_MONO) { - events(FlutterError(code: "-3", - message: "Currently only ChannelConfig CHANNEL_IN_MONO is supported", details:nil)) - return nil - } - fallthrough - case 2: - SAMPLE_RATE = config[1]!; - fallthrough - case 1: - AUDIO_SOURCE = AudioSource(rawValue:config[0]!)!; - if(AUDIO_SOURCE != AudioSource.DEFAULT) { - events(FlutterError(code: "-3", - message: "Currently only default AUDIO_SOURCE (id: 0) is supported", details:nil)) - return nil - } - default: - events(FlutterError(code: "-3", - message: "At least one argument (AudioSource) must be provided ", details:nil)) - return nil - } - NSLog("Setting eventSinkn: \(config.count)"); - self.eventSink = events; - startCapture(); - return nil; + // argument check + let config = arguments as! [Int?]; + NSLog("received config \(config)") + if ( + config.count == 4 && + config[0] == 0 && // audio source must be DEFAULT + config[1] == 48000 && // sampleRate must be 48000 as tested on my machine + config[2] == 16 && // channel config must be MONO + config[3] == 2 // audio format must be ENCODING_PCM_16BIT + ) {} else { + NSLog("warning: configuration not supported. The only supported config is (DEFAULT, 48000, MONO, 16BIT) ") } - - func startCapture() { - if let audioCaptureDevice : AVCaptureDevice = AVCaptureDevice.default(for:AVMediaType.audio) { - - self.session = AVCaptureSession() - do { - try audioCaptureDevice.lockForConfiguration() - - let audioInput = try AVCaptureDeviceInput(device: audioCaptureDevice) - audioCaptureDevice.unlockForConfiguration() - - if(self.session.canAddInput(audioInput)){ - self.session.addInput(audioInput) - } - - - //let numChannels = CHANNEL_CONFIG == ChannelConfig.CHANNEL_IN_MONO ? 1 : 2 - // setting the preferred sample rate on AVAudioSession doesn't magically change the sample rate for our AVCaptureSession - // try AVAudioSession.sharedInstance().setPreferredSampleRate(Double(SAMPLE_RATE)) - - // neither does setting AVLinearPCMBitDepthKey on audioOutput.audioSettings (unavailable on iOS) - // 99% sure it's not possible to set streaming sample rate/bitrate - // try AVAudioSession.sharedInstance().setPreferredOutputNumberOfChannels(numChannels) - let audioOutput = AVCaptureAudioDataOutput() - audioOutput.setSampleBufferDelegate(self, queue: DispatchQueue.global()) - - if(self.session.canAddOutput(audioOutput)){ - self.session.addOutput(audioOutput) - } - - DispatchQueue.main.async { - self.session.startRunning() - } - } catch let e { - self.eventSink!(FlutterError(code: "-3", - message: "Error encountered starting audio capture, see details for more information.", details:e)) - } - } + + + let input = audioEngine.inputNode + let busID = 0 + let inputFormat = input.inputFormat(forBus: busID) + + sampleRate = inputFormat.sampleRate + + + input.installTap(onBus: busID, bufferSize: 512, format: inputFormat) { (buffer, time) in + guard let channelData = buffer.floatChannelData?[0] else { return } + + let floatArray = Array(UnsafeBufferPointer(start: channelData, count: Int(buffer.frameLength))) + //// used to findout the range of sample. it is even broader than -2 ... 2 + // NSLog("max \(floatArray.max()!) min \(floatArray.min()!)") + var intArray = floatArray.map { val in + // clamp the val to -2.0 ... 2.0 + let clamped = min(max(-2.0, val), 2.0) + return Int16(clamped * 16383) + } + //// use the following to get length information + // NSLog("\(intArray.count)") + // NSLog("\(buffer.frameLength)") + + intArray.withUnsafeMutableBytes { unsafeMutableRawBufferPointer in + let nBytes = Int(buffer.frameLength) * MemoryLayout.size + let unsafeMutableRawPointer = unsafeMutableRawBufferPointer.baseAddress! + + let data = Data(bytesNoCopy: unsafeMutableRawPointer, count: nBytes, deallocator: .none) + events(FlutterStandardTypedData(bytes: data)) + } } + + try! audioEngine.start() + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + NSLog("audio engine canceled"); - public func captureOutput(_ output : AVCaptureOutput, - didOutput sampleBuffer: CMSampleBuffer, - from connection : AVCaptureConnection) { - - let format = CMSampleBufferGetFormatDescription(sampleBuffer)! - let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(format)!.pointee - - let nChannels = Int(asbd.mChannelsPerFrame) // probably 2 - let bufferlistSize = AudioBufferList.sizeInBytes(maximumBuffers: nChannels) - let audioBufferList = AudioBufferList.allocate(maximumBuffers: nChannels) - for i in 0..? = CMAudioFormatDescriptionGetStreamBasicDescription(fd!) - self.actualSampleRate = asbd.mSampleRate - self.actualBitDepth = asbd.mBitsPerChannel - } - - let data = Data(bytesNoCopy: audioBufferList.unsafePointer.pointee.mBuffers.mData!, count: Int(audioBufferList.unsafePointer.pointee.mBuffers.mDataByteSize), deallocator: .none) - self.eventSink!(FlutterStandardTypedData(bytes: data)) + audioEngine.stop() + audioEngine = AVAudioEngine() - } + isRecording = false; + return nil + } } diff --git a/pubspec.yaml b/pubspec.yaml index 27c44e3..331b467 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,11 +1,11 @@ name: mic_stream description: A plugin to receive raw byte streams from a device's microphone. Audio is returned as `Stream`. -version: 0.6.1 +version: 0.7.2 homepage: https://github.com/anarchuser/mic_stream environment: - sdk: '>=2.12.0 <3.0.0' - flutter: ">=2.8.0" + sdk: '^3.0.0' + flutter: '>=3.13.5' module: androidX: true @@ -13,7 +13,7 @@ module: dependencies: flutter: sdk: flutter - permission_handler: ^9.2.0 + permission_handler: '^11.0.0' # For information on the generic Dart part of this file, see the # following page: https://www.dartlang.org/tools/pub/pubspec diff --git a/test/mic_stream_test.dart b/test/mic_stream_test.dart index 4a37a08..7d89be3 100644 --- a/test/mic_stream_test.dart +++ b/test/mic_stream_test.dart @@ -1,5 +1,5 @@ -import 'package:flutter/services.dart'; -import 'package:mic_stream/mic_stream.dart'; +// import 'package:flutter/services.dart'; +// import 'package:mic_stream/mic_stream.dart'; void main() { }