pocketsphinx Audio Recognition and Programming API

[twobob]

PHP Code:

#!/bin/sh

QUOT='{"titleBar":{"clientParams":{"secondary":"'
SECOND='","useDefaultPrimary":false}}}'"'"

# SETUP SOME PATHS
LOGFILE="/var/tmp/commands"
SETPROP="/usr/bin/lipc-set-prop"
EXTDIR="/mnt/us/extensions/pocketsphinx"
BINARY=$EXTDIR"/usr/local/bin/pocketsphinx_continuous"
LOOPER=$EXTDIR"/looper.sh"
BB="/mnt/us/extensions/system/bin/busybox"

# SETUP STARTUP ANNOUNCEMENT
STARTUPANNOUNCE="This is Vague. The Talking Gooey"

# DETERMINE MACHINE TYPE 
ARCH="`uname -m`"

# LONGEST SENTENCE (like "I[ ]AM[ ]A[ ]SENTENCE[ ]")
SPACEMAX="8"

# CLEAR DOWN ANY OLD JUNK
echo -n "" > /var/tmp/talk
echo -n "" >/var/tmp/wordmatch
echo -n "" >/var/tmp/command

# FORCE 1st loop
echo "$STARTUPANNOUNCE" > "$LOGFILE"

# START LOOP
while :; 
do usleep 50000
OUT=$(tail -n1 "$LOGFILE"  | tr '\n' ' ' )
SPACECOUNT=$(echo "$OUT" | grep -o " " | wc -l)
#echo -n $OUT

TEST=$( echo "$OUT" | tr -d ' '| tr -d '.')
#echo "TEST is --(""$TEST"")--"
#echo "COMMAND PARSED WAS --(""$OUT"")--"

# DO WE HAVE A COMMAND (remove whitespace for the test)
if [ -n "$TEST" ]; then

# IS A JUMBLE OF WORDS OR LESS THAN 3 WORDS
if [ ! "$SPACECOUNT" -le "$SPACEMAX" ]; then
echo  "Space count $SPACECOUNT Not Less than Maximum spaces allowed $SPACEMAX"
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Phrase too long... Take it slow""$SECOND"
OUT=""
/usr/bin/aplay "$EXTDIR""/usr/on1.wav"
fi

echo -n " $OUT ." > /var/tmp/talk 
echo -n "$OUT" > /var/tmp/command

VAR=$(cat /var/tmp/talk)
TRIMVAR=$(echo "$OUT" | sed 's/ *$//g')

# IS IT A VALID NON-SPACE ONLY VARIABLE?
if [ -n "$TRIMVAR"  ]; then

# RUN POSSIBLE ACTION MATCHING

echo "Case was --(""$TRIMVAR"")--"
echo -n "TRUE" > /var/tmp/wordmatch
case $TRIMVAR in

"$STARTUPANNOUNCE")\
        ( echo "RECOGNITION INTIALISED" ) ;;

# BACK TO THE HOME PAGE...
"HOME"|"GO HOME"|"CLOSE"|"CLOSE THAT"|"CLOSE IT"|"KILL IT"|"KILL THAT"|"KILL"|"HOME SCREEN")\
        ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.booklet.home'; ) ;;

# NOT IMPLEMENTED YET...
#"NEXT PAGE"|"PAGE FORWARD"|"FORWARD"|"PAGE RIGHT")\
#        (  "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.booklet.home' ) ;;

# EXAMPLE NATIVE APPLICATION LAUNCH
"BROWSER"|"WEB BROWSER"|"W_W_W"|"WORLD WIDE WEB"|"WEB")\
        ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.browser' ) ;;

# EXAMPLE ALTERNATE NATIVE APPLICATION LAUNCH
"OPEN STORE"|"AMAZON STORE"|"OPEN AMAZON STORE"|"BUY BOOKS"|"OPEN AMAZON BOOK STORE")\
        ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.store' ) ;;

# EXAMPLE KINDLET LAUNCH
"RUN COOL"|"OPEN COOL"|"SELECT COOL")\
        (  echo -n "KINDLET" > /var/tmp/wordmatch  ) ;;  # They only have 5 seconds to start up so lets really free up resources

# EXAMPLE BACKGROUNDED GTK APPLICATION
"RUN KAY TERM"|"OPEN KAY TERM"|"KAY TERM")\
        (  /mnt/us/extensions/kterm/bin/runit.sh & sleep 2;   ) ;;  # This might need a moment or two...

# LONG RUNNING LOCKING TASK
#"TIME"|"WHATS THE TIME"|"READ ME THE TIME")\
#        (  /mnt/us/extensions/flite/run_voice.sh ;sleep 1 ) ;;  # This could be a locking task... have to think

# LONG RUNNING LOCKING SHELL TASK
#"VAGUE HELP"|"HELP ME VAUGE"|"READ ME THE VAGUE HELP"|"READ ME THE VAGUE HELP TEXT"|"READ VAGUE HELP"|"READ THE VAGUE HELP TEXT" |"VAGUE HELP ME"|"HELP VAGUE")\
#        (  echo -n "HELPME" > /var/tmp/wordmatch  ) ;;  # This could be a locking task... have to think

# EXAMPLE BACKGROUND SHELL TAKS
"OPEN NETWORKING"|"RUN NETWORKING"|"U_S_B NETWORKING"|"RUN U_S_B NETWORKING"|"NETWORK"|"NETWORKING"|"RUN NETWORK")\
        ( /mnt/us/usbnet/bin/usbnetwork ) ;;

# EXAMPLE LIPC CALL
"SCREEN SAVER"|"POWER OFF"|"POWER DOWN")\
        ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 0; /usr/bin/powerd_test -p ) ;;

# EXAMPLE STATEFUL CONTROL
"KILL SCREEN SAVER")\
        ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 1;  ) ;;

# EXAMPLE OTHER STATEFUL CONTROL... NOT YET IMPLEMENTED
#"KILL SCREEN SAVER")\
#        ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 1;  ) ;;

# CLOSE THE APPLICATION
"SHUT DOWN")\
        ( echo -n "QUIT" > /var/tmp/wordmatch ) ;;

*) ( echo -n "FALSE" > /var/tmp/wordmatch ; echo "Case was $OUT is not processed" )  ;;
esac

# NOW WE WILL BREAK OUT OF CASE MATCHING TO DO SECONDARY TASKS...
# YEAH YEAH some of this could be cased. MEH.

MATCH=$( cat /var/tmp/wordmatch )
echo "Words found as command Match = $MATCH"

if [ "$MATCH" == "KINDLET" ] ; then
killall -9 pocketsphinx_continuous 
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""STARTING KUAL!""$SECOND"
usleep 30000;
"$SETPROP" com.lab126.appmgrd start "app://com.lab126.booklet.kindlet/mnt/us/documents/KindleLauncher-2.0.azw2"
sleep 5
fi

# READ OUT THE HELP
#if [ "$MATCH" == "HELPME" ] ; then
#killall -9 pocketsphinx_continuous 
#"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""HELP INFORMATION""$SECOND" 
#/mnt/us/extensions/flite/flite "$EXTDIR""/usr/vague"
#sleep 8
#fi

# EXIT THE APPLICATION
if [ "$MATCH" == "QUIT" ] ; then
killall -9 pocketsphinx_continuous 
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Goodbye""$SECOND" 
exit 
fi

if [ "$MATCH" == "TRUE" ] ; then

# shut down the listener
killall -9 pocketsphinx_continuous
usleep 10000

# IS THIS A HIGHER DEVICE?
if [ "$ARCH" == "armv7l" ] ; then

lipc-set-prop com.lab126.pillow configureChrome -s "$QUOT""$OUT.""$SECOND"  
fi

#echo "WORDS ARE  `cat /var/tmp/talk`"
# Announce Recognised Words
/mnt/us/extensions/flite/flite -f /var/tmp/talk &
sleep 1

# START LISTENER AGAIN
"$LOOPER" &
else
# ALERT BAD MATCH
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Phase not recognised.""$SECOND"
/usr/bin/aplay "$EXTDIR""/usr/on3.wav"
# End actions on MATCHED
fi

echo -n '' > /var/tmp/commands
sleep 2
fi

# TIDY UP AGAIN ?
echo -n "" > /var/tmp/talk
echo -n "" > /var/tmp/command
echo -n "" > "$LOGFILE" 

# IS THIS A HIGHER DEVICE?
if [ "$ARCH" == "armv7l" ] ; then
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Now Listening.""$SECOND"

# CLOSE LOOPS
fi
# INDICATE READINESS... RESET SOUND CARD?
/usr/bin/aplay "$EXTDIR""/usr/on2.wav"
fi
done 


This is pretty solid now and I have demonstrated a fair few techniques here.

I'll look to package this for the touch as-is as an alpha and then back port.

The code is already built to run on the 3's.

[twobob]

Write up: http://kual.knetconnect.com/vague-vo...-gui-for-kual/

[eureka]

@twobob, wonderful, awesome!

[knc1]

Looking real good.

Only thing I noticed missing was that the application (Sphinx) can support "voice training" (perhaps intended for a later example).

Edit:
Well, since I mentioned it, I will include the link here - -
Which IS NOT a promise that the Kindle version will ever be capable of being trained.
http://cmusphinx.sourceforge.net/wiki/tutorialam

Start your timer now -
See how long it takes the Duokan crew to learn it also understands Chinese and steal the demo from us from you.

[twobob]

http://cmusphinx.sourceforge.net/201...ith-cmusphinx/

: ) That was nice of them

[knc1]

Originally Posted by twobob

http://cmusphinx.sourceforge.net/201...ith-cmusphinx/

: ) That was nice of them

Since they linked to our KUAL site with the phrase: "Already available" - -

It might be a good idea to either attach the files to our announcement post on the KUAl support forum site or a link to the attachments in our KUAL Wiki.

Or at least a "file package being prepared" notice somewhere.

(Sorry Alex, we needed slightly different support resources than you have here - we had to build our own sites to support the KUAL activity.)

[twobob]

yeah. The files ARE available on GITHUB.

I'll look to publish today then. Just going to add a little bit more "bullet-proofing", by the time hackaday subsumes it we should be good to go.

thanks for the tip Mike.

[knc1]

Originally Posted by twobob

yeah. The files ARE available on GITHUB.
- - - -
thanks for the tip Mike.

Yes, I understand that but readers who 'link in' may not.

I also understand it was late at night for me and even 6 hours later for you - -
So we may have missed a few things (like a link to the binaries, )

I will do my part in keeping an eye on the server resources in use and purchase more if required.
(rant) to all the people who think this "free Internet" stuff is $0.00 cost . . . . (/rant)

[twobob]

Okay so I added "proper" test for "READY TO RECEIVE command" initialisation.

And hopefully cleaner coding. More comments etc...

Rerolled the second set of tests back into a case selection where possible.

There should be fewer pops and crackles now as the engine is not shut down unnecessarily so often...

PHP Code:

#!/bin/sh
# MAKE PUTTING THINGS IN THE STATUS BAR SANELY BEARABLE
QUOT='{"titleBar":{"clientParams":{"secondary":"'
SECOND='","useDefaultPrimary":false}}}'"'"

# SETUP SOME PATHS
LOGFILE="/var/tmp/commands"
SETPROP="/usr/bin/lipc-set-prop"
EXTDIR="/mnt/us/extensions/vague"
BINARY=$EXTDIR"/bin/pocketsphinx_continuous"
LOOPER=$EXTDIR"/looper.sh"
BB="/mnt/us/extensions/system/bin/busybox"

# DETERMINE MACHINE TYPE 
ARCH="`uname -m`"

# NOW SETUP A MACHINE SPECIFIC BIN PATH
SYSTEMBIN="/mnt/us/extensions/system/bin/$ARCH"

# SETUP A QUICK KILL COMMAND
SHUSH="killall -9 pocketsphinx_continuous"

# LONGEST SENTENCE (like "I[ ]AM[ ]A[ ]SENTENCE[ ]")
SPACEMAX="8"

# CLEAR DOWN ANY OLD JUNK
killall -9 looper.sh
$SHUSH
echo -n "" > /var/tmp/talk
echo -n "" >/var/tmp/command
echo -n "" >/var/tmp/wordmatch
WORDMATCH="FALSE"  # CREATE DEFAULT NON-MATCH


# SETUP STARTUP ANNOUNCEMENT  ==========================================================
STARTUPANNOUNCE="This is Vague. The Talking Gooey"
# YOU CAN EDIT THIS TO YOUR DESIRES ====================================================

"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""$STARTUPANNOUNCE""$SECOND" 

# FORCE 1st loop
/mnt/us/extensions/flite/flite " $STARTUPANNOUNCE ." 
sleep 2


# LETS GET THIS PARTY STARTED
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Initialising.""$SECOND"

# STARTUP OUR FIRST BACKGROUND LISTENER
"$LOOPER" &

# START LOOP
while :; 
do usleep 50000
OUT=$(tail -n1 "$LOGFILE"  | tr '\n' ' ' )

#echo "out is $OUT"

# Work our way back down the list in reverse.
"$SYSTEMBIN"/truncate -s -"$(tail -n1 $LOGFILE | wc -c)" "$LOGFILE"


SPACECOUNT=$(echo "$OUT" | grep -o " " | wc -l)
#echo -n $OUT

TEST=$( echo "$OUT" | tr -d ' '| tr -d '.')
#echo "TEST is --(""$TEST"")--"
#echo "COMMAND PARSED WAS --(""$OUT"")--"

# DO WE HAVE A COMMAND (remove whitespace for the test)
if [ -n "$TEST" ]; then

# IS A JUMBLE OF WORDS OR LESS THAN 3 WORDS
if [ ! "$SPACECOUNT" -le "$SPACEMAX" ]; then
echo  "Space count $SPACECOUNT Not Less than Maximum spaces allowed $SPACEMAX"
"$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Phrase too long... Take it slow""$SECOND"
OUT=""
/usr/bin/aplay "$EXTDIR""/usr/on1.wav"
fi

echo -n " $OUT ." > /var/tmp/talk 
echo -n "$OUT" > /var/tmp/command

VAR=$(cat /var/tmp/talk)
TRIMVAR=$(echo "$OUT" | sed 's/ *$//g')

# IS IT A VALID NON-SPACE ONLY VARIABLE?
if [ -n "$TRIMVAR"  ]; then

# RUN POSSIBLE ACTION MATCHING

###################################################################################################
######  YOU PROBABLY WANT TO ADD TO THESE COMMANDS WITH YOUR OWN

echo "Case was --(""$TRIMVAR"")--"
echo -n "TRUE" > /var/tmp/wordmatch

        case $TRIMVAR in
            
        "READY") ( echo -n "STARTUP" > /var/tmp/wordmatch ) ;;
        
        # BACK TO THE HOME PAGE...
        "HOME"|"GO HOME"|"CLOSE"|"CLOSE THAT"|"CLOSE IT"|"KILL IT"|"KILL THAT"|"KILL"|"HOME SCREEN")\
                ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.booklet.home'; killall runit.sh; killall -9 kterm ) ;;
        
        # NOT IMPLEMENTED YET...
        #"NEXT PAGE"|"PAGE FORWARD"|"FORWARD"|"PAGE RIGHT")\
        #        (  "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.booklet.home' ) ;;
        
        # EXAMPLE NATIVE APPLICATION LAUNCH
        "BROWSER"|"WEB BROWSER"|"W_W_W"|"WORLD WIDE WEB"|"WEB")\
                ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.browser' ) ;;
        
        # EXAMPLE ALTERNATE NATIVE APPLICATION LAUNCH
        "OPEN STORE"|"AMAZON STORE"|"OPEN AMAZON STORE"|"BUY BOOKS"|"OPEN AMAZON BOOK STORE")\
                ( "$SETPROP" -s com.lab126.appmgrd start 'app://com.lab126.store' ) ;;
        
        # EXAMPLE KINDLET LAUNCH
        "RUN COOL"|"OPEN COOL"|"SELECT COOL")\
                ( echo -n "KINDLET" > /var/tmp/wordmatch  ) ;;  # They only have 5 seconds to start up so lets really free up resources
        
        # EXAMPLE BACKGROUNDED GTK APPLICATION
        "RUN KAY TERM"|"OPEN KAY TERM"|"KAY TERM")\
                (  /mnt/us/extensions/kterm/bin/runit.sh & sleep 2;   ) ;;  # This might need a moment or two...
        
        # EXAMPLE BACKGROUND SHELL TASK
        "OPEN NETWORKING"|"RUN NETWORKING"|"U_S_B NETWORKING"|"RUN U_S_B NETWORKING"|"NETWORK"|"NETWORKING"|"RUN NETWORK")\
                ( /mnt/us/usbnet/bin/usbnetwork ) ;;
        
        # EXAMPLE LIPC CALL
        "SCREEN SAVER"|"POWER OFF"|"POWER DOWN")\
                ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 0; /usr/bin/powerd_test -p ) ;;
        
        # EXAMPLE STATEFUL CONTROL
        "KILL SCREEN SAVER")\
                ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 1;  ) ;;
        
        # EXAMPLE OTHER STATEFUL CONTROL... NOT YET IMPLEMENTED
        #"KILL SOMETHING ELSE")\
        #        ( "$SETPROP" -i com.lab126.powerd preventScreenSaver 1;  ) ;;
        
        # CLOSE THE APPLICATION
        "SHUT DOWN")\
                ( echo -n "QUIT" > /var/tmp/wordmatch ) ;;
        
        *) ( echo -n "FALSE" > /var/tmp/wordmatch ; echo "Case was $OUT is not processed" )  ;;
        esac

#################################################################################################################
## EVERYTHING BELOW HERE YOU PROBABLY DO NOT NEED TO CHANGE

# NOW WE WILL BREAK OUT OF CASE MATCHING TO DO SECONDARY TASKS...
# YEAH YEAH some of this could be cased. - EDIT: Now it is.

WORDMATCH=$( cat /var/tmp/wordmatch )
echo "Words found as command Match = $WORDMATCH"


        if [ "$WORDMATCH" == "QUIT" ] ; then
        $SHUSH
        /mnt/us/extensions/flite/flite " Goodbye."
        # EXIT THE APPLICATION
        
        "$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Goodbye""$SECOND" 
        exit 
        fi

        case $WORDMATCH in
        
        'KINDLET' ) (  $SHUSH
        "$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""STARTING KUAL!""$SECOND"
        usleep 30000;
        "$SETPROP" com.lab126.appmgrd start "app://com.lab126.booklet.kindlet/mnt/us/documents/KindleLauncher-2.0.azw2"
        sleep 5
        
        # Announce Recognised Words
        /mnt/us/extensions/flite/flite " Cool has been started."
        sleep 1
        
        # RESTART OUR BACKGROUND LISTENER
        "$LOOPER" &        
        
        
        ) ;;
        
        'TRUE' ) (
        
        # shut down the listener
        $SHUSH
        usleep 10000
        
            # IS THIS A HIGHER DEVICE?
            if [ "$ARCH" == "armv7l" ] ; then
            lipc-set-prop com.lab126.pillow configureChrome -s "$QUOT""$OUT.""$SECOND"  
            fi
        
        #echo "WORDS ARE  `cat /var/tmp/talk`"
        # Announce Recognised Words
        /mnt/us/extensions/flite/flite -f /var/tmp/talk &
        sleep 1
        
        # RESTART OUR BACKGROUND LISTENER
        "$LOOPER" &
        
        ) ;;
        
        'STARTUP' ) (
            
            echo "READY!!"
            # IS THIS A HIGHER DEVICE?
            if [ "$ARCH" == "armv7l" ] ; then
            "$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Now Listening.""$SECOND"
            fi
        
        ) ;;
        
          * )  (
          
        echo "NO MATCH was --(""$TRIMVAR"")--"
        "$SETPROP" -s com.lab126.pillow configureChrome -s "$QUOT""Phrase not recognised.""$SECOND"
        /usr/bin/aplay "$EXTDIR""/usr/on3.wav"  
          
          )  ;; # no-op ; 
        esac  
        
# END IF VALID PHRASE test
sleep 2
fi

# TIDY UP AGAIN ?
echo -n "" > /var/tmp/talk
echo -n "" > /var/tmp/command
#echo -n "" > "$LOGFILE" 


fi
done 


I think this is ready for general release now rather than just via github

[twobob]

Okay All packaged up for the touch. Several small improvements completed.

http://kual.knetconnect.com/vague-0....dded-flite.zip

If there IS enough interest I repack it for the 3.

Cheers.

[twobob]

I forgot to bundle the armv7l truncate from kual-system. ill do it tomorrow

Okay Okay I did it today.

: ) Bundle fixed. Remember this one is for the KINDLE TOUCH.

Just saying it again.

I bundled the truncate, fixed the menu.json paths and threw in a spangly new busybox (just for fun as penance)

Enjoy. Feel free to do tests.

[thomass]

Originally Posted by twobob

Okay All packaged up for the touch.

thanks, will try.

[silver18]

This is awesome!!!
I'm playing around with this (using it to make an april fool to my mom!).
I tried adding Komic...
I can hijack other commands (browser for example) but I can't add my own.

This is what I added:

# EXAMPLE OPEN KOMIC
"OPEN COMIC"|"COMIC")\
	( /mnt/us/extensions/komic/bin/resource/Scripts/start.sh & sleep 2; ) ;;

I always get Phrase not recognised (I even tried Google Translate to get a nice pronunciation)

Spoiler:

This is super-awesome! I could send some commands to Komic to turn page, open menu etc...
Basically, I could even provide a full control over Komic, it's just a matter of adding commands to your script and sending thing to Komic to trigger javascripts...

You're my hero!!!!

[knc1]

You also need to extend the VR vocabulary file with the word / phrase "OPEN COMIC" and "COMIC"

Scroll down this page, past the "goodbye" example to find the link to the on-line dictionary/vocabulary file builder:
http://kual.knetconnect.com/vague-vo...-gui-for-kual/

Then edit the vocabulary file (may only be in the github repo - I didn't look) with the new word(s) and phrase(s) - upload to the knowledge base builder found from the link on above page.

Note: Builder also does Chinese and French, not just English

[silver18]

Originally Posted by knc1

You also need to extend the VR vocabulary file with the word / phrase "OPEN COMIC" and "COMIC"

Scroll down this page, past the "goodbye" example to find the link to the on-line dictionary/vocabulary file builder:
http://kual.knetconnect.com/vague-vo...-gui-for-kual/

Then edit the vocabulary file (may only be in the github repo - I didn't look) with the new word(s) and phrase(s) - upload to the knowledge base builder found from the link on above page.

Note: Builder also does Chinese and French, not just English

Ahhhh, this makes sense now!!
I found everything, I just need to find the original vocabulary file from VAGUE to add my own words...

Once again, thanks knc1!!!