diff --git a/ai-conversation/Makefile b/ai-conversation/Makefile new file mode 100644 index 0000000..1f4f296 --- /dev/null +++ b/ai-conversation/Makefile @@ -0,0 +1,14 @@ +CC = gcc +CFLAGS = -Wall -g +TARGET = audio_capture +SRC = audio_capture.c +LIBS = -lavdevice -lavformat -lavcodec -lavutil -lasound + +$(TARGET): $(SRC) + $(CC) $(CFLAGS) -o $(TARGET) $(SRC) $(LIBS) + +clean: + rm -f *.o $(TARGET) + +install-deps: + sudo apt install build-essential libavformat-dev libavdevice-dev libavcodec-dev libavutil-dev libasound2-dev diff --git a/ai-conversation/README.md b/ai-conversation/README.md new file mode 100644 index 0000000..29daba2 --- /dev/null +++ b/ai-conversation/README.md @@ -0,0 +1,12 @@ +# Conversation +This proof of concept is for implementing conversational loop with AI at a low level. + +- [x] Capture audio +- [ ] Use deepgram for live transcription +- [ ] Detect final speech/turn detection +- [ ] AI answer -> text +- [ ] Text -> generated voice with deepgram +- [ ] Playback on client side + +## Requirements +Platform: the project is meant for Linux, specifically Ubuntu desktop. diff --git a/ai-conversation/audio_capture b/ai-conversation/audio_capture new file mode 100755 index 0000000..99d283e Binary files /dev/null and b/ai-conversation/audio_capture differ diff --git a/ai-conversation/audio_capture.c b/ai-conversation/audio_capture.c new file mode 100644 index 0000000..e823048 --- /dev/null +++ b/ai-conversation/audio_capture.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include + +// NOTE: find this via `arecord -l` +#define ALSA_MIC_HARDWARE_DEVICE "hw:3" + +void print_pcm_state(snd_pcm_t *pcm_handle) { + snd_pcm_state_t state = snd_pcm_state(pcm_handle); + const char *state_name = snd_pcm_state_name(state); + printf("State of pcm handle: %s\n", state_name); +} + +int main() { + snd_pcm_t *pcm_handle = NULL; + + int err; + err = snd_pcm_open(&pcm_handle, ALSA_MIC_HARDWARE_DEVICE, SND_PCM_STREAM_CAPTURE, 0); + if (err < 0) { + printf("unable to open pcm device"); + return -1; + } + print_pcm_state(pcm_handle); + + const char *name = snd_pcm_name(pcm_handle); + printf("found this device: %s\n", name); + + snd_pcm_info_t *pcm_info; + snd_pcm_info_malloc(&pcm_info); + err = snd_pcm_info(pcm_handle, pcm_info); + if (err < 0) { + snd_pcm_close(pcm_handle); + printf("unable to get info of pcm device"); + return -1; + } + name = snd_pcm_info_get_name(pcm_info); + if (err < 0) { + printf("unable to get card info"); + snd_pcm_close(pcm_handle); + snd_pcm_info_free(pcm_info); + return -1; + } + printf("got pcm name: %s \n", name); + + print_pcm_state(pcm_handle); + snd_pcm_info_free(pcm_info); + + snd_pcm_hw_params_t *params; + err = snd_pcm_hw_params_malloc(¶ms); + if (err < 0) { + fprintf(stderr, "unable to mallow hw params"); + return -1; + } + err = snd_pcm_hw_params_any(pcm_handle, params); + if (err < 0) { + fprintf(stderr, "unable to find ranges for hw device"); + snd_pcm_close(pcm_handle); + snd_pcm_hw_params_free(params); + return -1; + } + + uint period_size = 1024; + uint buffer_size = 4096; + uint channel_count = 1; + uint bytes_per_sample = 2; // FORMAT_S16_LE + uint samples_per_second = 48000; + snd_pcm_hw_params_set_access(pcm_handle, params, SND_PCM_ACCESS_RW_INTERLEAVED); + snd_pcm_hw_params_set_channels(pcm_handle, params, channel_count); + snd_pcm_hw_params_set_buffer_size(pcm_handle, params, buffer_size); + snd_pcm_hw_params_set_format(pcm_handle, params, SND_PCM_FORMAT_S16_LE); + snd_pcm_hw_params_set_rate(pcm_handle, params, samples_per_second, 0); + snd_pcm_hw_params_set_periods(pcm_handle, params, period_size, 0); + + err = snd_pcm_hw_params(pcm_handle, params); + if (err < 0) { + fprintf(stderr, "unable to prepare pcm\n"); + snd_pcm_close(pcm_handle); + snd_pcm_hw_params_free(params); + return -1; + } + + print_pcm_state(pcm_handle); + + err = snd_pcm_start(pcm_handle); + if (err < 0) { + fprintf(stderr, "unable to start pcm"); + snd_pcm_close(pcm_handle); + snd_pcm_hw_params_free(params); + return -1; + } + + print_pcm_state(pcm_handle); + + const uint seconds_to_capture = 5; + const uint periods_to_read = (samples_per_second * seconds_to_capture) / period_size; + char *buf = malloc(period_size * channel_count * bytes_per_sample); + uint periods_read = 0; + /* Read one period at a time from the hardware buffer + * We set the hardware buffer to read 4096 frames. + * The period size is 1024 frames. + * Each period is 2048 BYTES because each frame is 2 bytes. + * Each frame is 2 bytes because of 1 channel, and 16 bit format + */ + while (snd_pcm_readi(pcm_handle, (void *)buf, period_size) > 0 && periods_read < periods_to_read) { + printf("======READING FRAMES========\n"); + + // Because each frame is 2 bytes (format/bit rate), cast to int16 + int16_t *samples = (int16_t *)buf; + for (int i = 0; i < period_size; i++) { + printf("%d ", samples[i]); + } + printf("\n"); + periods_read++; + } + + free((void *)buf); + snd_pcm_close(pcm_handle); + snd_pcm_hw_params_free(params); + + return 0; +} diff --git a/ai-conversation/main.c b/ai-conversation/main.c new file mode 100644 index 0000000..6c4d7f9 --- /dev/null +++ b/ai-conversation/main.c @@ -0,0 +1,5 @@ +#include + +int main() { + printf("hello world \n"); +}