Things used in this project

Schematics

Google Home Demonstration
google_home_video_D7xOH8NV4w.mp4
Google Home Layout
Google home layout 0txlieulzl

Code

Speech to Light Command DecodingJava
/*
 * Copyright 2017, The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.example.androidthings.assistant;

import android.app.Activity;
import android.content.SharedPreferences;
import android.media.AudioFormat;
import android.os.Bundle;
import android.os.Handler;
import android.preference.PreferenceManager;
import android.util.Log;
import android.widget.ArrayAdapter;
import android.widget.ListView;
import com.example.androidthings.assistant.EmbeddedAssistant.ConversationCallback;
import com.example.androidthings.assistant.EmbeddedAssistant.RequestCallback;
import com.google.android.things.contrib.driver.button.Button;
import com.google.android.things.contrib.driver.voicehat.VoiceHat;
import com.google.android.things.pio.Gpio;
import com.google.android.things.pio.PeripheralManagerService;
import com.google.android.things.pio.UartDevice;
import com.google.assistant.embedded.v1alpha1.ConverseResponse.EventType;
import com.google.auth.oauth2.UserCredentials;
import com.google.rpc.Status;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONException;

public class AssistantActivity extends Activity implements Button.OnButtonEventListener {
    private static final String TAG = AssistantActivity.class.getSimpleName();

    /***
     * Peripheral and drivers constants.
     ***/
    private static final boolean AUDIO_USE_I2S_VOICEHAT_IF_AVAILABLE = false;
    private static final int BUTTON_DEBOUNCE_DELAY_MS = 20;

    /***
     * Audio constants.
     ***/
    private static final String PREF_CURRENT_VOLUME = "current_volume";
    private static final int SAMPLE_RATE = 16000;
    private static final int ENCODING = AudioFormat.ENCODING_PCM_16BIT;
    private static final int DEFAULT_VOLUME = 100;

    private static final AudioFormat AUDIO_FORMAT_STEREO =
            new AudioFormat.Builder()
                    .setChannelMask(AudioFormat.CHANNEL_IN_STEREO)
                    .setEncoding(ENCODING)
                    .setSampleRate(SAMPLE_RATE)
                    .build();

    /***
     * Hardware peripherals.
     ***/
    private VoiceHat mVoiceHat;
    private Button mButton;
    private Gpio mLed;
    private UartDevice mUART;

    private Handler mMainHandler;

    /***
     * List & adapter to store and display the history of Assistant Requests.
     ***/
    private EmbeddedAssistant mEmbeddedAssistant;
    private ArrayList<String> mAssistantRequests = new ArrayList<>();
    private ArrayAdapter<String> mAssistantRequestsAdapter;

    /***
     * UART DEVICE NAME
     ***/
    private static final String UART_DEVICE_NAME = "UART6";

    /***
     * Start of Google Home Code
     ***/

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        Log.i(TAG, "starting Google Home demo");

        /*** Create GUI View for Google Home Demo ***/
        setContentView(R.layout.activity_main);
        ListView assistantRequestsListView = (ListView) findViewById(R.id.assistantRequestsListView);
        mAssistantRequestsAdapter =
                new ArrayAdapter<>(this, android.R.layout.simple_list_item_1,
                        mAssistantRequests);
        assistantRequestsListView.setAdapter(mAssistantRequestsAdapter);
        mMainHandler = new Handler(getMainLooper());

        /*** Setup peripherals ***/
        try {
            if (AUDIO_USE_I2S_VOICEHAT_IF_AVAILABLE) {
                PeripheralManagerService pioService = new PeripheralManagerService();
                List<String> i2sDevices = pioService.getI2sDeviceList();
                if (i2sDevices.size() > 0) {
                    try {
                        Log.i(TAG, "creating voice hat driver");
                        mVoiceHat = new VoiceHat(
                                BoardDefaults.getI2SDeviceForVoiceHat(),
                                BoardDefaults.getGPIOForVoiceHatTrigger(),
                                AUDIO_FORMAT_STEREO
                        );
                        mVoiceHat.registerAudioInputDriver();
                        mVoiceHat.registerAudioOutputDriver();
                    } catch (IllegalStateException e) {
                        Log.w(TAG, "Unsupported board, falling back on default audio device:", e);
                    }
                }
            }
            mButton = new Button(BoardDefaults.getGPIOForButton(),
                    Button.LogicState.PRESSED_WHEN_LOW);
            mButton.setDebounceDelay(BUTTON_DEBOUNCE_DELAY_MS);
            mButton.setOnButtonEventListener(this);
            PeripheralManagerService pioService = new PeripheralManagerService();
            mLed = pioService.openGpio(BoardDefaults.getGPIOForLED());
            mLed.setDirection(Gpio.DIRECTION_OUT_INITIALLY_LOW);
            mLed.setActiveType(Gpio.ACTIVE_LOW);

            /*** Setup UART Peripheral ***/
            try {
                PeripheralManagerService manager = new PeripheralManagerService();
                mUART = manager.openUartDevice(UART_DEVICE_NAME);
            } catch (IOException e) {
                Log.w(TAG, "Unable to access UART device", e);
            }
            configureUartFrame(mUART);

        } catch (IOException e) {
            Log.e(TAG, "error configuring peripherals:", e);
            return;
        }

        /*** Set volume from preferences ***/
        SharedPreferences preferences = PreferenceManager.getDefaultSharedPreferences(this);
        int initVolume = preferences.getInt(PREF_CURRENT_VOLUME, DEFAULT_VOLUME);
        Log.i(TAG, "setting audio track volume to: " + initVolume);

        /*** Verifying Credentials ***/
        UserCredentials userCredentials = null;
        try {
            userCredentials =
                    EmbeddedAssistant.generateCredentials(this, R.raw.credentials);
        } catch (IOException | JSONException e) {
            Log.e(TAG, "error getting user credentials", e);
        }


        mEmbeddedAssistant = new EmbeddedAssistant.Builder()
                .setCredentials(userCredentials)
                .setAudioSampleRate(SAMPLE_RATE)
                .setAudioVolume(initVolume)
                .setRequestCallback(new RequestCallback() {
                    @Override
                    public void onRequestStart() {
                        Log.i(TAG, "starting assistant request, enable microphones");
                    }

                    @Override
                    public void onSpeechRecognition(String utterance) {
                        Log.i(TAG, "assistant request text: " + utterance);
                        try {
                            processSpeech(utterance);
                        } catch (IOException e) {
                            Log.i(TAG, "Can not process:" + utterance);
                        }
//                        mMainHandler.post(new Runnable() {
//                            @Override
//                            public void run() {
//                                mAssistantRequestsAdapter.add(utterance);
//                            }
//                        });
                    }
                })
                .setConversationCallback(new ConversationCallback() {
                    @Override
                    public void onConversationEvent(EventType eventType) {
                        Log.d(TAG, "converse response event: " + eventType);
                    }

                    @Override
                    public void onAudioSample(ByteBuffer audioSample) {
                        if (mLed != null) {
                            try {
                                mLed.setValue(!mLed.getValue());
                            } catch (IOException e) {
                                Log.w(TAG, "error toggling LED:", e);
                            }
                        }
                    }

                    @Override
                    public void onConversationError(Status error) {
                        Log.e(TAG, "converse response error: " + error);
                    }

                    @Override
                    public void onError(Throwable throwable) {
                        Log.e(TAG, "converse error:", throwable);
                    }

                    @Override
                    public void onVolumeChanged(int percentage) {
                        Log.i(TAG, "assistant volume changed: " + percentage);
                        // Update our shared preferences
                        SharedPreferences.Editor editor = PreferenceManager
                                .getDefaultSharedPreferences(AssistantActivity.this)
                                .edit();
                        editor.putInt(PREF_CURRENT_VOLUME, percentage);
                        editor.apply();
                    }

                    @Override
                    public void onConversationFinished() {
                        Log.i(TAG, "assistant conversation finished");
                        if (mLed != null) {
                            try {
                                mLed.setValue(false);
                            } catch (IOException e) {
                                Log.e(TAG, "error turning off LED:", e);
                            }
                        }
                    }
                })
                .build();
        mEmbeddedAssistant.connect();
    }

    @Override
    public void onButtonEvent(Button button, boolean pressed) {
        try {
            if (mLed != null) {
                mLed.setValue(pressed);
            }
        } catch (IOException e) {
            Log.d(TAG, "error toggling LED:", e);
        }
        if (pressed) {
            mEmbeddedAssistant.startConversation();
        }
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        Log.i(TAG, "destroying assistant demo");
        if (mLed != null) {
            try {
                mLed.close();
            } catch (IOException e) {
                Log.w(TAG, "error closing LED", e);
            }
            mLed = null;
        }
        if (mButton != null) {
            try {
                mButton.close();
            } catch (IOException e) {
                Log.w(TAG, "error closing button", e);
            }
            mButton = null;
        }
        if (mVoiceHat != null) {
            try {
                mVoiceHat.unregisterAudioOutputDriver();
                mVoiceHat.unregisterAudioInputDriver();
                mVoiceHat.close();
            } catch (IOException e) {
                Log.w(TAG, "error closing voice hat driver", e);
            }
            mVoiceHat = null;
        }
        if (mUART != null) {
            try {
                mUART.close();
                mUART = null;
            } catch (IOException e) {
                Log.w(TAG, "Unable to close UART device", e);
            }
        }
        mEmbeddedAssistant.destroy();
    }

    public void configureUartFrame(UartDevice uart) throws IOException {
        // Configure the UART port
        uart.setBaudrate(9600);
        uart.setDataSize(8);
        uart.setParity(UartDevice.PARITY_NONE);
        uart.setStopBits(1);
    }

    public void writeUartData(UartDevice uart, byte data) throws IOException {
        byte[] buffer = {data};
        int count = uart.write(buffer, buffer.length);
        Log.d(TAG, "Wrote " + count + " bytes to peripheral");
    }

    /***
     * Defines start byte and end byte to know when transmission is started and ended
     ***/
    private static final byte START_BYTE = (byte) 56;
    private static final byte END_BYTE = (byte) 40;

    /***
     * Defines different types of messages
     ***/
    private static final byte LIGHT_MESSAGE = (byte)  01;
    private static final byte DOOR_MESSAGE = (byte) 02;
    private static final byte MODE_MESSAGE = (byte) 03;

    /***
     * Defines byte codes for various rooms
     ***/
    private static final byte DINING_ROOM = (byte) 01;
    private static final byte LIVING_ROOM = (byte) 02;
    private static final byte FOYER_ROOM = (byte) 03;
    private static final byte BED_ROOM = (byte) 04;
    private static final byte OFFICE_ROOM = (byte) 05;
    private static final byte ALL = (byte) 06;

    /***
     * Defines byte codes for different doors
     ***/
    private static final byte FRONT_DOOR = (byte) 01;
    private static final byte GARAGE_DOOR = (byte) 02;

    /***
     * Defines byte codes for various modes
     ***/
    private static final byte DANCE_MODE = (byte) 01;
    private static final byte CHRISTMAS_MODE = (byte) 02;

    public void processSpeech(String utterance) throws IOException {
        writeUartData(mUART, START_BYTE);
        if (utterance.contains("light")) {
            writeUartData(mUART, LIGHT_MESSAGE);
            boolean found_room = false;

            if (utterance.contains("dining")) {
                writeUartData(mUART, DINING_ROOM);
                found_room = true;
            } else if (utterance.contains("living")) {
                writeUartData(mUART, LIVING_ROOM);
                found_room = true;
            } else if (utterance.contains("foyer")) {
                writeUartData(mUART, FOYER_ROOM);
                found_room = true;
            } else if (utterance.contains("bed")) {
                writeUartData(mUART, BED_ROOM);
                found_room = true;
            } else if (utterance.contains("office")) {
                writeUartData(mUART, OFFICE_ROOM);
                found_room = true;
            } else if (utterance.contains("all")) {
                writeUartData(mUART, ALL);
                found_room = true;
            }

            if (found_room) {
                if (utterance.contains("red")) {
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 0);
                } else if (utterance.contains("green")) {
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 0);
                } else if (utterance.contains("blue")) {
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 1);
                } else if (utterance.contains("purple")) {
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 1);
                } else if (utterance.contains("yellow")) {
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 0);
                } else if (utterance.contains("on")) {
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 1);
                    writeUartData(mUART, (byte) 1);
                } else if (utterance.contains("off")) {
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 0);
                    writeUartData(mUART, (byte) 0);
                }
            }
        } else if (utterance.contains("door")) {
            writeUartData(mUART, DOOR_MESSAGE);
            writeUartData(mUART, FRONT_DOOR);

            if (utterance.contains("open")) {
                writeUartData(mUART, (byte) 1);
            } else if (utterance.contains("close")) {
                writeUartData(mUART, (byte) 0);
            }
        } else if (utterance.contains("mode")) {
            writeUartData(mUART, MODE_MESSAGE);

            if (utterance.contains("dance")) {
                writeUartData(mUART, DANCE_MODE);
            } else if (utterance.contains("christmas")) {
                writeUartData(mUART, CHRISTMAS_MODE);
            }
        }
        writeUartData(mUART, END_BYTE);
    }
}

Credits

Head shot ewiocnchlh
Tanner Smith
1 project • 0 followers
I am a Junior Electrical and Computer Engineer and Computer Science Minor at Rowan University.
Contact

Replications

Did you replicate this project? Share it!

I made one

Love this project? Think it could be improved? Tell us what you think!

Give feedback

Comments

Add projectSign up / Login