Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version

Speech Recognition

, 26 Feb 2004
Voice-activated OS
using System;
using System.Drawing;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Data;
using System.Runtime.InteropServices;
using SpeechLib;
using System.Text;

namespace SR
{
	/// <summary>
	/// This is the main form
	/// It's responsible for creating all necessary objects for speech recognition
	/// </summary>
	public class frmMain : System.Windows.Forms.Form
	{
		/// <summary>
		/// API functions
		/// </summary>
		[DllImport("user32.Dll")]
		public static extern int keybd_event(byte ch,byte scan,int flag,int info);

		[DllImport("user32.dll")]
		static extern IntPtr GetForegroundWindow();

		[DllImport("user32.Dll")]
		public static extern IntPtr GetMenu(IntPtr hwnd);
		
		[DllImport("user32.dll")]
		static extern IntPtr GetSubMenu(IntPtr hMnu, int nPos);

		[DllImport("user32.dll")]
		static extern int GetMenuItemCount(IntPtr hMnu);

		[DllImport("user32.dll")]
		static extern int GetMenuString(IntPtr hMnu, int uIDItem, StringBuilder text, int nMaxCount, int uFlag);

		[DllImport("user32.dll")]
		static extern int GetCursorPos(out Point pnt);

		[DllImport("user32.dll")]
		static extern int GetMenuState(IntPtr hMnu, int uIDItem, int uFlags);

		///<summary>
		///Global variables
		///</summary>

		//struct for phrase list
		//used by the function SAPIGrammarFromArrayList(ArrayList phraseList)
		private struct command
		{
			public string ruleName;
			public string phrase;
		}

		//SAPI
		private SpeechLib.SpSharedRecoContext		objRecoContext;
		private SpeechLib.ISpeechRecoGrammar		grammar;
		private SpeechLib.ISpeechGrammarRule		rule=null;
		private SpeechLib.ISpeechGrammarRuleState   state;

		//character
		private AgentObjects.IAgentCtlCharacterEx agent1;
		Point pntMouse=new Point(0,0);

		//forms
		//they needed to be global couse they could be created and closed in different places
		frmAbout frmAbout1=null;
		frmCommands frmCommands1=null;
		frmFavorites frmFavorites1=null;
		frmAccuracyChange frmAccuracyChange1=null;
		frmProfileChange frmProfileChange1=null;

		//misc
		string appPath="";
		byte keyHolding=0;
		int accuracyLimit=0;
		float accuracyMax=0.1F;						//to avoid divide by zero
		string firstRecognition="";
		string previousGrammar="XMLActivate.xml";	//for safty load something...

		/// <summary>
		/// Form components
		/// </summary>
		private AxAgentObjects.AxAgent axAgent1;
		private System.Windows.Forms.ProgressBar MicVolume;
		private System.Windows.Forms.PictureBox pictureBox1;
		private System.Windows.Forms.ContextMenu contextMenu1;
		private System.Windows.Forms.MenuItem menuItem1;
		private System.Windows.Forms.MenuItem menuItem2;
		private System.ComponentModel.IContainer components;
		private System.Windows.Forms.Timer timer1;
		private System.Windows.Forms.MenuItem menuItem3;
		private System.Windows.Forms.MenuItem menuItem5;
		private System.Windows.Forms.MenuItem menuItem6;
		private System.Windows.Forms.Timer timer2;
		private System.Windows.Forms.MenuItem menuItem7;
		private System.Windows.Forms.MenuItem menuItem8;
		private System.Windows.Forms.MenuItem menuItem9;
		private System.Windows.Forms.MenuItem menuItem10;
		private System.Windows.Forms.MenuItem menuItem12;
		private System.Windows.Forms.MenuItem menuItem13;
		private System.Windows.Forms.MenuItem menuItem4;
		private System.Windows.Forms.Label label1;
		private System.Windows.Forms.Label label2;
		private System.Windows.Forms.MenuItem menuItem11;
		private System.Windows.Forms.MenuItem menuItem14;
		private System.Windows.Forms.OpenFileDialog openFileDialog1;

		public frmMain()
		{
			//
			// Required for Windows Form Designer support
			//
			InitializeComponent();

			//
			// TODO: Add any constructor code after InitializeComponent call
			//
		}

		/// <summary>
		/// Clean up any resources being used.
		/// </summary>
		protected override void Dispose( bool disposing )
		{
			if( disposing )
			{
				if (components != null) 
				{
					components.Dispose();
				}
			}
			base.Dispose( disposing );
		}

		#region Windows Form Designer generated code
		/// <summary>
		/// Required method for Designer support - do not modify
		/// the contents of this method with the code editor.
		/// </summary>
		private void InitializeComponent()
		{
			this.components = new System.ComponentModel.Container();
			System.Resources.ResourceManager resources = new System.Resources.ResourceManager(typeof(frmMain));
			this.axAgent1 = new AxAgentObjects.AxAgent();
			this.MicVolume = new System.Windows.Forms.ProgressBar();
			this.contextMenu1 = new System.Windows.Forms.ContextMenu();
			this.menuItem3 = new System.Windows.Forms.MenuItem();
			this.menuItem5 = new System.Windows.Forms.MenuItem();
			this.menuItem4 = new System.Windows.Forms.MenuItem();
			this.menuItem6 = new System.Windows.Forms.MenuItem();
			this.menuItem7 = new System.Windows.Forms.MenuItem();
			this.menuItem12 = new System.Windows.Forms.MenuItem();
			this.menuItem10 = new System.Windows.Forms.MenuItem();
			this.menuItem9 = new System.Windows.Forms.MenuItem();
			this.menuItem8 = new System.Windows.Forms.MenuItem();
			this.menuItem13 = new System.Windows.Forms.MenuItem();
			this.menuItem1 = new System.Windows.Forms.MenuItem();
			this.menuItem2 = new System.Windows.Forms.MenuItem();
			this.pictureBox1 = new System.Windows.Forms.PictureBox();
			this.openFileDialog1 = new System.Windows.Forms.OpenFileDialog();
			this.timer1 = new System.Windows.Forms.Timer(this.components);
			this.timer2 = new System.Windows.Forms.Timer(this.components);
			this.label1 = new System.Windows.Forms.Label();
			this.label2 = new System.Windows.Forms.Label();
			this.menuItem11 = new System.Windows.Forms.MenuItem();
			this.menuItem14 = new System.Windows.Forms.MenuItem();
			((System.ComponentModel.ISupportInitialize)(this.axAgent1)).BeginInit();
			this.SuspendLayout();
			// 
			// axAgent1
			// 
			this.axAgent1.Enabled = true;
			this.axAgent1.Location = new System.Drawing.Point(48, 40);
			this.axAgent1.Name = "axAgent1";
			this.axAgent1.OcxState = ((System.Windows.Forms.AxHost.State)(resources.GetObject("axAgent1.OcxState")));
			this.axAgent1.Size = new System.Drawing.Size(32, 32);
			this.axAgent1.TabIndex = 0;
			this.axAgent1.TabStop = false;
			this.axAgent1.Visible = false;
			this.axAgent1.ClickEvent += new AxAgentObjects._AgentEvents_ClickEventHandler(this.axAgent1_ClickEvent);
			this.axAgent1.DragComplete += new AxAgentObjects._AgentEvents_DragCompleteEventHandler(this.axAgent1_DragComplete);
			this.axAgent1.DragStart += new AxAgentObjects._AgentEvents_DragStartEventHandler(this.axAgent1_DragStart);
			// 
			// MicVolume
			// 
			this.MicVolume.ContextMenu = this.contextMenu1;
			this.MicVolume.Location = new System.Drawing.Point(8, 136);
			this.MicVolume.Name = "MicVolume";
			this.MicVolume.Size = new System.Drawing.Size(112, 16);
			this.MicVolume.TabIndex = 3;
			// 
			// contextMenu1
			// 
			this.contextMenu1.MenuItems.AddRange(new System.Windows.Forms.MenuItem[] {
																						 this.menuItem3,
																						 this.menuItem11,
																						 this.menuItem14,
																						 this.menuItem5,
																						 this.menuItem4,
																						 this.menuItem6,
																						 this.menuItem7,
																						 this.menuItem12,
																						 this.menuItem10,
																						 this.menuItem9,
																						 this.menuItem8,
																						 this.menuItem13,
																						 this.menuItem1,
																						 this.menuItem2});
			// 
			// menuItem3
			// 
			this.menuItem3.Index = 0;
			this.menuItem3.Text = "Stop listen";
			this.menuItem3.Click += new System.EventHandler(this.menuItem3_Click);
			// 
			// menuItem5
			// 
			this.menuItem5.Index = 3;
			this.menuItem5.Text = "-";
			// 
			// menuItem4
			// 
			this.menuItem4.Index = 4;
			this.menuItem4.Text = "Add favorites";
			this.menuItem4.Click += new System.EventHandler(this.menuItem4_Click);
			// 
			// menuItem6
			// 
			this.menuItem6.Index = 5;
			this.menuItem6.Text = "Change character";
			this.menuItem6.Click += new System.EventHandler(this.menuItem6_Click);
			// 
			// menuItem7
			// 
			this.menuItem7.Index = 6;
			this.menuItem7.Text = "Change accuracy limit";
			this.menuItem7.Click += new System.EventHandler(this.menuItem7_Click);
			// 
			// menuItem12
			// 
			this.menuItem12.Index = 7;
			this.menuItem12.Text = "-";
			// 
			// menuItem10
			// 
			this.menuItem10.Index = 8;
			this.menuItem10.Text = "Change user profile";
			this.menuItem10.Click += new System.EventHandler(this.menuItem10_Click);
			// 
			// menuItem9
			// 
			this.menuItem9.Index = 9;
			this.menuItem9.Text = "Mic training wizard...";
			this.menuItem9.Click += new System.EventHandler(this.menuItem9_Click);
			// 
			// menuItem8
			// 
			this.menuItem8.Index = 10;
			this.menuItem8.Text = "User training wizard...";
			this.menuItem8.Click += new System.EventHandler(this.menuItem8_Click);
			// 
			// menuItem13
			// 
			this.menuItem13.Index = 11;
			this.menuItem13.Text = "Profile proporties wizard...";
			this.menuItem13.Click += new System.EventHandler(this.menuItem13_Click);
			// 
			// menuItem1
			// 
			this.menuItem1.Index = 12;
			this.menuItem1.Text = "-";
			// 
			// menuItem2
			// 
			this.menuItem2.Index = 13;
			this.menuItem2.Text = "Exit";
			this.menuItem2.Click += new System.EventHandler(this.menuItem2_Click);
			// 
			// pictureBox1
			// 
			this.pictureBox1.BackColor = System.Drawing.Color.Navy;
			this.pictureBox1.ContextMenu = this.contextMenu1;
			this.pictureBox1.Location = new System.Drawing.Point(0, 128);
			this.pictureBox1.Name = "pictureBox1";
			this.pictureBox1.Size = new System.Drawing.Size(128, 64);
			this.pictureBox1.TabIndex = 4;
			this.pictureBox1.TabStop = false;
			// 
			// openFileDialog1
			// 
			this.openFileDialog1.Filter = "Agent *.acs|*.acs";
			// 
			// timer1
			// 
			this.timer1.Interval = 10;
			this.timer1.Tick += new System.EventHandler(this.timer1_Tick_1);
			// 
			// timer2
			// 
			this.timer2.Interval = 500;
			this.timer2.Tick += new System.EventHandler(this.timer2_Tick);
			// 
			// label1
			// 
			this.label1.BackColor = System.Drawing.Color.Navy;
			this.label1.ContextMenu = this.contextMenu1;
			this.label1.ForeColor = System.Drawing.SystemColors.HighlightText;
			this.label1.Location = new System.Drawing.Point(8, 158);
			this.label1.Name = "label1";
			this.label1.Size = new System.Drawing.Size(112, 16);
			this.label1.TabIndex = 7;
			this.label1.Text = "label1";
			// 
			// label2
			// 
			this.label2.BackColor = System.Drawing.Color.Navy;
			this.label2.ContextMenu = this.contextMenu1;
			this.label2.ForeColor = System.Drawing.SystemColors.HighlightText;
			this.label2.Location = new System.Drawing.Point(8, 173);
			this.label2.Name = "label2";
			this.label2.Size = new System.Drawing.Size(112, 16);
			this.label2.TabIndex = 8;
			this.label2.Text = "Accuracy 0%";
			// 
			// menuItem11
			// 
			this.menuItem11.Index = 1;
			this.menuItem11.Text = "-";
			// 
			// menuItem14
			// 
			this.menuItem14.Checked = true;
			this.menuItem14.Index = 2;
			this.menuItem14.Text = "Use agent";
			this.menuItem14.Click += new System.EventHandler(this.menuItem14_Click);
			// 
			// frmMain
			// 
			this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
			this.BackColor = System.Drawing.Color.FromArgb(((System.Byte)(255)), ((System.Byte)(192)), ((System.Byte)(255)));
			this.ClientSize = new System.Drawing.Size(128, 192);
			this.Controls.AddRange(new System.Windows.Forms.Control[] {
																		  this.label2,
																		  this.label1,
																		  this.MicVolume,
																		  this.axAgent1,
																		  this.pictureBox1});
			this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.None;
			this.Icon = ((System.Drawing.Icon)(resources.GetObject("$this.Icon")));
			this.Name = "frmMain";
			this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen;
			this.Text = "Speech Recognition";
			this.TopMost = true;
			this.TransparencyKey = System.Drawing.Color.FromArgb(((System.Byte)(255)), ((System.Byte)(192)), ((System.Byte)(255)));
			this.Load += new System.EventHandler(this.frmMain_Load);
			((System.ComponentModel.ISupportInitialize)(this.axAgent1)).EndInit();
			this.ResumeLayout(false);

		}
		#endregion

		/// <summary>
		/// The main entry point for the application.
		/// </summary>
		[STAThread]
		static void Main() 
		{
			Application.Run(new frmMain());
		}

		/// <summary>
		/// Create the main object (SpSharedRecoContext), init the gramar
		/// And activating the events ...
		/// 1.Recognition	=> if any phrase recognized
		/// 2.AudioLevel	=> to watch the mic volume
		/// </summary>
		private void initSAPI()
		{
			try
			{
				objRecoContext = new SpeechLib.SpSharedRecoContext();
				objRecoContext.AudioLevel+= new _ISpeechRecoContextEvents_AudioLevelEventHandler(RecoContext_VUMeter);
				objRecoContext.Recognition+= new _ISpeechRecoContextEvents_RecognitionEventHandler(RecoContext_Recognition);
				objRecoContext.EventInterests=SpeechLib.SpeechRecoEvents.SRERecognition | SpeechLib.SpeechRecoEvents.SREAudioLevel;

				//create grammar interface with ID = 0
				grammar=objRecoContext.CreateGrammar(0);
			}
			catch(Exception ex)
			{
				MessageBox.Show("Exception \n"+ex.ToString(),"Error - initSAPI");
			}
		}

		/// <summary>
		/// Loading grammar from file
		/// And resetting all previous rules (delete all previous rules !)
		/// </summary>
		/// <param name="FileName">FilName add to the global variable appPath</param>
		private void SAPIGrammarFromFile(string FileName)
		{
			try
			{
				grammar.CmdLoadFromFile(appPath+FileName,SpeechLib.SpeechLoadOption.SLODynamic);
				grammar.CmdSetRuleIdState(0,SpeechRuleState.SGDSActive);

				//save as previous grammar for commands list
				if (FileName != "XMLCommands.xml")
					previousGrammar=FileName;
			}
			catch
			{
                MessageBox.Show("Error loading file "+FileName+"\n","Error - SAPIGrammarFromFile");
			}
		}

		/// <summary>
		/// Adding new ruleName and phrase programmatically (not from grammar file)
		/// To an existing grammar (Adding to previous rules ! without deleting them)
		/// Can be add rules to exist grammar from file or from first added rules
		/// </summary>
		/// <param name="phraseList">
		/// ArrayList (phraseList) contains struct command objects that contains ...
		/// 1. ruleName		(rule Name must be individual)
		/// 2. phrase		(word to be recognized)
		/// </param>
		private void SAPIGrammarFromArrayList(ArrayList phraseList)
		{
			object propertyValue="";
			command command1;

			int i;
			for (i=0;i<phraseList.Count;i++)
			{
				command1=(command)phraseList[i];

				//add new rule with ID = i+100
				rule=grammar.Rules.Add(command1.ruleName, SpeechRuleAttributes.SRATopLevel, i+100);

				//add new word to the rule
				state=rule.InitialState;
				propertyValue="";
				state.AddWordTransition(null,command1.phrase," ", SpeechGrammarWordType.SGLexical, "", 0, ref propertyValue, 1F);

				//commit rules
				grammar.Rules.Commit();

				//make rule active (needed for each rule)
				grammar.CmdSetRuleState(command1.ruleName, SpeechRuleState.SGDSActive);
			}
		}

		/// <summary>
		/// main objRecoContext event
		/// launched when engine recognized a phrase
		/// </summary>
		/// <param name="e">contained information on the phrase that been recognized</param>
		public void RecoContext_Recognition(int StreamNumber, object StreamPosition, SpeechRecognitionType RecognitionType,	ISpeechRecoResult e)
		{
			//calculate accuracy
			float accuracy=(float)e.PhraseInfo.Elements.Item(0).EngineConfidence;

			//change accuracyMax dynamicly
			if (accuracyMax<accuracy)
				accuracyMax=accuracy;

			if (accuracy<0)
				accuracy=0;

			accuracy=(int)((float)accuracy/accuracyMax*100);
			label2.Text="Accuracy "+accuracy.ToString()+ "%";

			//get phrase
			string phrase=e.PhraseInfo.GetText(0,-1,true);
			//make sure it's in lower case (for safer use only)
			phrase=phrase.ToLower();

			//if recognized any ...
			if (phrase!="" && accuracy>=accuracyLimit)
			{
				//Only if agent enabled
				if (menuItem14.Checked==true)
				{
					agent1.StopAll("");
					agent1.Speak(phrase,"");
				}

				switch (e.PhraseInfo.Rule.Name)		//rule name (not the phrase !)
				{
					case "Activate":
					{
						//Only if agent enabled
						if (menuItem14.Checked==true)
						{
							//show character
							agent1.Show(false);
						}

						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";

						//Only if agent enabled
						if (menuItem14.Checked==true)
						{
							//animate character
							agent1.Play("StartListening");
							agent1.Speak("I'm listening","");
						}
						break;
					}
					case "Deactivate":
					{
						//load grammar
						SAPIGrammarFromFile("XMLActivate.xml");

						//notify user
						label1.Text="Deactivate";

						//Only if agent enabled
						if (menuItem14.Checked==true)
						{
							//animate character
							agent1.Play("Wave");
							agent1.Hide(false);
						}
						break;
					}
					case "Start":
					{
						keybd_event((byte)Keys.LWin,0,0,0);	//key down
						keybd_event((byte)Keys.LWin,0,2,0);	//key up

						//load grammar
						SAPIGrammarFromFile("XMLStart.xml");

						//notify user
						label1.Text="Start";
						break;
					}
					case "Right":
					{
						keybd_event((byte)Keys.Right,0,0,0);	//key down
						keybd_event((byte)Keys.Right,0,2,0);	//key up
						break;
					}
					case "Left":
					{
						keybd_event((byte)Keys.Left,0,0,0);	//key down
						keybd_event((byte)Keys.Left,0,2,0);	//key up
						break;
					}
					case "Up":
					{
						keybd_event((byte)Keys.Up,0,0,0);		//key down
						keybd_event((byte)Keys.Up,0,2,0);		//key up
						break;
					}
					case "Down":
					{
						keybd_event((byte)Keys.Down,0,0,0);	//key down
						keybd_event((byte)Keys.Down,0,2,0);	//key up
						break;
					}
					case "Enter":
					{
						keybd_event((byte)Keys.Enter,0,0,0);	//key down
						keybd_event((byte)Keys.Enter,0,2,0);	//key up
						break;
					}
					case "Escape":
					{
						keybd_event((byte)Keys.Escape,0,0,0);	//key down
						keybd_event((byte)Keys.Escape,0,2,0);	//key up
						keybd_event((byte)Keys.LMenu,0,2,0);	//key up

						//load grammar (used to reset grammar in case it contains menu stuff ...)
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}
					case "PureEscape":
					{
						keybd_event((byte)Keys.Escape,0,0,0);	//key down
						keybd_event((byte)Keys.Escape,0,2,0);	//key up
						break;
					}
					case "Alt":
					{
						keybd_event((byte)Keys.LMenu,0,0,0);	//key down
						keybd_event((byte)Keys.LMenu,0,2,0);	//key up

						//check if there is any menu and hook it
						IntPtr hWnd=GetForegroundWindow();
						IntPtr hMnu=GetMenu(hWnd);
						int mnuCnt=GetMenuItemCount(hMnu);

						if (mnuCnt!=0)
						{
							//Only if agent enabled
							if (menuItem14.Checked==true)
							{
								//animate character
								agent1.Play("DoMagic1");
								agent1.Think("Hooking menu ...");
							}

							//add menu to grammar
							hookMenu(hMnu);

							//Only if agent enabled
							if (menuItem14.Checked==true)
							{
								//animate character
								agent1.Play("Idle1_1");
							}
						}
						else
						{
							//load grammar
							SAPIGrammarFromFile("XMLDeactivate.xml");

							//notify user
							label1.Text="Activate";
						}
						break;
					}
					case "Tab":
					{
						keybd_event((byte)Keys.Tab,0,0,0);	//key down
						keybd_event((byte)Keys.Tab,0,2,0);	//key up
						break;
					}
					case "ShiftTab":
					{
						keybd_event((byte)Keys.LShiftKey,0,0,0);	//key down
						keybd_event((byte)Keys.Tab,0,0,0);		//key down
						keybd_event((byte)Keys.Tab,0,2,0);		//key up
						keybd_event((byte)Keys.LShiftKey,0,2,0);	//key up
						break;
					}
					case "CloseProgram":
					{
						Close();
						break;
					}
					case "ShowAbout":
					{
						if (frmAbout1==null)
						{
							//show frmAbout
							frmAbout1=new frmAbout();
							frmAbout1.Closed+=new EventHandler(frmAbout1_Closed);
							//send user profile
							frmAbout1.Tag=(string)objRecoContext.Recognizer.Profile.GetDescription(0);
							frmAbout1.Show();
						}

						//load grammar
						SAPIGrammarFromFile("XMLAbout.xml");

						//notify user
						label1.Text="About Speech Recognition";
						break;
					}
					case "CloseAbout":
					{
						//close frmAbout
						if (frmAbout1!=null)
						{
							frmAbout1.Close();
							frmAbout1=null;
						}
						break;
					}
					case "ShowCommands":
					{
						if (frmCommands1==null)
						{
							//show frmAbout
							frmCommands1=new frmCommands();
							frmCommands1.Closed+=new EventHandler(frmCommands1_Closed);
							//send grammar
							frmCommands1.Tag=label1.Text;
							frmCommands1.Show();
						}

						//load grammar
						SAPIGrammarFromFile("XMLCommands.xml");
						break;
					}
					case "CloseCommands":
					{
						//close frmCommands
						if (frmCommands1!=null)
						{
							frmCommands1.Close();
							frmCommands1=null;
						}
						break;
					}
					case "ShowFavorites":
					{
						if (frmFavorites1==null)
						{
							//show frmFavorites
							frmFavorites1=new frmFavorites();
							frmFavorites1.Closed+=new EventHandler(frmFavorites1_Closed);
							//send file name
							frmFavorites1.Tag=appPath+"XMLFavorites.xml";
							frmFavorites1.Show();
						}

						//load grammar
						SAPIGrammarFromFile("XMLFavorites.xml");

						//notify user
						label1.Text="Favorites";
						break;
					}
					case "CloseFavorites":
					{
						//show frmAbout
						if (frmFavorites1!=null)
						{
							frmFavorites1.Close();
							frmFavorites1=null;
						}
						break;
					}
					case "CloseForm":
					{
						IntPtr hWnd=GetForegroundWindow();

						//make sure we are not closing our program ...
						if (hWnd!=this.Handle)
						{
							keybd_event((byte)Keys.LMenu,0,0,0);	//key down
							keybd_event((byte)Keys.F4,0,0,0);		//key down
							keybd_event((byte)Keys.LMenu,0,2,0);	//key up
							keybd_event((byte)Keys.F4,0,2,0);		//key up
						}
						break;
					}
					case "Programs":
					case "Documents":
					case "Settings":
					case "Search":
					case "Help":
					case "Run":
					{
						keybd_event((byte)(e.PhraseInfo.Rule.Name[0]),0,0,0);	//key down
						keybd_event((byte)(e.PhraseInfo.Rule.Name[0]),0,2,0);	//key up

						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}
					case "RunProgram":
					{
						//show frmAbout
						if (frmFavorites1!=null)
						{
							frmFavorites1.Close();
							frmFavorites1=null;
						}

						try
						{
							System.Diagnostics.Process.Start(phrase);
						}
						catch
						{
							//Only if agent enabled
							if (menuItem14.Checked==true)
							{
								agent1.Speak("Could not run : "+phrase,"");
							}
						}

						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}
					case "SwitchProgram":
					{
						keybd_event((byte)Keys.LMenu,0,0,0);	//key down
						keybd_event((byte)Keys.Tab,0,0,0);	//key down
						keybd_event((byte)Keys.Tab,0,2,0);	//key up

						//load grammar
						SAPIGrammarFromFile("XMLSwitchProgram.xml");

						//notify user
						label1.Text="Switch Program";
						break;
					}
					case "SwitchEnter":
					{
						keybd_event((byte)Keys.LMenu,0,2,0);	//key up
						
						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}

					case "HoldKey":
					{
						//load grammar
						SAPIGrammarFromFile("XMLStickyKeys.xml");

						//notify user
						label1.Text="Press key";
						break;
					}

					case "ReleaseKey":
					{
						timer2.Enabled=false;

						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}

					case "HoldRight":
					{
						keyHolding=(byte)Keys.Right;
						timer2.Enabled=true;
						break;
					}
					case "HoldLeft":
					{
						keyHolding=(byte)Keys.Left;
						timer2.Enabled=true;
						break;
					}
					case "HoldUp":
					{
						keyHolding=(byte)Keys.Up;
						timer2.Enabled=true;
						break;
					}
					case "HoldDown":
					{
						keyHolding=(byte)Keys.Down;
						timer2.Enabled=true;
						break;
					}
					case "PageUp":
					{
						keybd_event((byte)Keys.PageUp,0,0,0);	//key down
						keybd_event((byte)Keys.PageUp,0,2,0);	//key up
						break;
					}
					case "Yes":
					{
						keybd_event((byte)Keys.Y,0,0,0);	//key down
						keybd_event((byte)Keys.Y,0,2,0);	//key up
						break;
					}
					case "No":
					{
						keybd_event((byte)Keys.N,0,0,0);	//key down
						keybd_event((byte)Keys.N,0,2,0);	//key up
						break;
					}
					case "BackSpace":
					{
						keybd_event((byte)Keys.Back,0,0,0);	//key down
						keybd_event((byte)Keys.Back,0,2,0);	//key up
						break;
					}
					case "ShutDown":
					{
						Shell32.ShellClass a=new Shell32.ShellClass();
						a.ShutdownWindows();
	
						//load grammar
						SAPIGrammarFromFile("XMLShutDown.xml");

						//notify user
						label1.Text="Shut Down";
						break;
					}
					case "ActivateWithoutAnimation":
					{
						//load grammar
						SAPIGrammarFromFile("XMLDeactivate.xml");

						//notify user
						label1.Text="Activate";
						break;
					}
					case "EnterNumericState":
					{
						//load grammar
						SAPIGrammarFromFile("XMLNumericState.xml");

						//notify user
						label1.Text="Numeric State...";
						break;
					}
					case "Zero":
					case "One":
					case "Two":
					case "Three":
					case "Four":
					case "Five":
					case "Six":
					case "Seven":
					case "Eight":
					case "Nine":
					{
						byte k=(byte)e.PhraseInfo.GetText(0,-1,false)[0];
                
						keybd_event((byte)(k+'0'),0,0,0);	//key down
						keybd_event((byte)(k+'0'),0,2,0);	//key up
						break;
					}
					case "Plus":
					{
						keybd_event((byte)Keys.Add,0,0,0);	//key down
						keybd_event((byte)Keys.Add,0,2,0);	//key up
						break;
					}
					case "Minus":
					{
						keybd_event((byte)Keys.Subtract,0,0,0);	//key down
						keybd_event((byte)Keys.Subtract,0,2,0);	//key up
						break;
					}
					case "Div":
					{
						keybd_event((byte)Keys.Divide,0,0,0);	//key down
						keybd_event((byte)Keys.Divide,0,2,0);	//key up
						break;
					}
					case "Mul":
					{
						keybd_event((byte)Keys.Multiply,0,0,0);	//key down
						keybd_event((byte)Keys.Multiply,0,2,0);	//key up
						break;
					}
					case "Equal":
					{
						keybd_event(187,0,0,0);	//key down
						keybd_event(187,0,2,0);	//key up
						break;
					}
					case "EnterAlphabeticState":
					{
						//load grammar
						SAPIGrammarFromFile("XMLAlphabeticState.xml");

						//notify user
						label1.Text="Alphabetic State...";
						break;
					}
					case "abcA":case "abcB":case "abcC":case "abcD":case "abcE":case "abcF":case "abcG":
					case "abcH":case "abcI":case "abcJ":case "abcK":case "abcL":case "abcM":case "abcN":
					case "abcO":case "abcP":case "abcQ":case "abcR":case "abcS":case "abcT":case "abcU":
					case "abcV":case "abcW":case "abcX":case "abcY":case "abcZ":
					{
						firstRecognition=phrase;
						string str1=phrase;
						str1=str1.ToUpper();
						keybd_event((byte)(str1[0]),0,0,0);	//key down
						keybd_event((byte)(str1[0]),0,2,0);	//key up
						break;
					}
					case "At":
					{
						keybd_event((byte)Keys.LShiftKey,0,0,0);	//key down
						keybd_event((byte)Keys.D2,0,0,0);			//key down
						keybd_event((byte)Keys.D2,0,2,0);			//key up
						keybd_event((byte)Keys.LShiftKey,0,2,0);	//key up
						break;
					}
					case "UnderLine":
					{
						keybd_event((byte)Keys.LShiftKey,0,0,0);	//key down
						keybd_event((byte)Keys.OemMinus,0,0,0);		//key down
						keybd_event((byte)Keys.OemMinus,0,2,0);		//key up
						keybd_event((byte)Keys.LShiftKey,0,2,0);	//key up
						break;
					}
					case "Dash":
					{
						keybd_event((byte)Keys.Subtract,0,0,0);		//key down
						keybd_event((byte)Keys.Subtract,0,2,0);		//key up
						break;
					}
					case "Dot":
					{
						keybd_event(190,0,0,0);	//key down
						keybd_event(190,0,2,0);	//key up
						break;
					}
					case "BackSlash":
					{
						keybd_event((byte)Keys.Divide,0,0,0);	//key down
						keybd_event((byte)Keys.Divide,0,2,0);	//key up
						break;
					}
					case "AlphabeticStateNo":
					{
						//delete the first letter
						keybd_event((byte)Keys.Back,0,0,0);	//key down
						keybd_event((byte)Keys.Back,0,2,0);	//key up

						//write the replacement letter
						string str1=firstRecognition;

						//fix miss recognition
						switch(firstRecognition)
						{
							case "a": str1="h"; break;
							case "b": str1="d"; break;
							case "c": str1="t"; break;
							case "d": str1="p"; break;
							case "f": str1="x"; break;
							case "h": str1="f"; break;
							case "m": str1="n"; break;
							case "n": str1="l"; break;
							case "l": str1="m"; break;
							case "p": str1="v"; break;
							case "u": str1="q"; break;
							case "v": str1="t"; break;
							case "e": str1="b"; break;
							case "j": str1="k"; break;
						}

						firstRecognition=str1;
						str1=str1.ToUpper();

						keybd_event((byte)(str1[0]),0,0,0);	//key down
						keybd_event((byte)(str1[0]),0,2,0);	//key up
						break;
					}
					
					//else press the key (probably a menu ...)
					default:
					{
						string str1=e.PhraseInfo.Rule.Name;
						str1=str1.ToUpper();

						keybd_event((byte)(str1[0]),0,0,0);	//key down
						keybd_event((byte)(str1[0]),0,2,0);	//key up

						//could be submenu (hook it)
						hookSubmenu(e.PhraseInfo.Rule.Name[0].ToString());
						break;
					}
				}
			}

			//if not recognized ...
			else
			{
				//Only if agent enabled
				if (menuItem14.Checked==true)
				{
					//animate character
					agent1.Play("Decline");
				}
			}
		}

		/// <summary>
		/// objRecoContext event
		/// launched every time there is change in the mic volume
		/// and updates the progressBar1.value
		/// </summary>
		/// <param name="e">current mic volume</param>
		public void RecoContext_VUMeter(int StreamNumber, object StreamPosition, int e)
		{
			MicVolume.Value=e;
		}

		private void frmMain_Load(object sender, System.EventArgs e)
		{
			//app path
			appPath=Application.StartupPath+@"\";	//to use from the debuger "\..\..\"

			//initialize SAPI objects
			initSAPI();

			//loading initial grammar
			SAPIGrammarFromFile("XMLActivate.xml");

			//notify user
            label1.Text="Deactivate";

			//reposition frmMain
			this.Left=(short)(Screen.GetBounds(this).Width-Width);
			this.Top=(short)(Screen.GetBounds(this).Height-Height-50);

			//loading character
			try
			{
				axAgent1.Characters.Load("agentAssistant",appPath+"merlin.acs");
				agent1=axAgent1.Characters.Character("agentAssistant");
			}
			catch
			{
				MessageBox.Show("Error loading character\nThe agent will be disabled...","Error - character");
				menuItem14.Checked=false;
			}

			//Only if agent enabled
			if (menuItem14.Checked==true)
			{
				//reposition character
				agent1.Left=(short)this.Left;
				agent1.Top=(short)this.Top;

				//show character
				agent1.Show(false);
				agent1.Play("Greet");
				agent1.Play("Explain");
				agent1.Speak("I'm under your control.","");
				agent1.Play("Acknowledge");
				agent1.Speak("just activate me when ever you want...","");
				agent1.Play("Wave");
				agent1.Hide(false);
			}
			
			//hide the program from the taskbar
			this.ShowInTaskbar=false;

			//Only if agent enabled
			if (menuItem14.Checked==true)
			{
				//Deactivate AutoPopupMenu, we'll use our context menu ...
				agent1.AutoPopupMenu=false;
			}
		}

		/// <summary>
		/// "exit" pressed from context menu
		/// </summary>
		private void menuItem2_Click(object sender, System.EventArgs e)
		{
			//close the program
			Close();
		}

		/// <summary>
		/// "Stop listen"/"Start listen" pressed from context menu
		/// </summary>
		private void menuItem3_Click(object sender, System.EventArgs e)
		{
			if (contextMenu1.MenuItems[0].Text=="Stop listen")
			{
				contextMenu1.MenuItems[0].Text="Start listen";
				label1.ForeColor=Color.Red;
				label2.ForeColor=Color.Red;

				//pause dictation
				grammar.State=SpeechLib.SpeechGrammarState.SGSDisabled;
			}
			else
			{
				contextMenu1.MenuItems[0].Text="Stop listen";
				label1.ForeColor=Color.White;
				label2.ForeColor=Color.White;

				//resume dictation
				grammar.State=SpeechLib.SpeechGrammarState.SGSEnabled;
			}
		}

		/// <summary>
		/// because agent1 dos not support context menu
		/// we show it in the main form in the clicked place
		/// </summary>
		private void axAgent1_ClickEvent(object sender, AxAgentObjects._AgentEvents_ClickEvent e)
		{
			if (e.button==2)
			{
				Point pnt=new Point(e.x-this.Left,e.y-this.Top);
				contextMenu1.Show(this,pnt);
			}
		}

		/// <summary>
		/// hooking menu and adding all items to the grammar
		/// </summary>
		/// <param name="hMnu">Menu handle</param>
		private void hookMenu(IntPtr hMnu)
		{
			//reset grammar
			SAPIGrammarFromFile("XMLDeactivate.xml");

			int mnuCnt=GetMenuItemCount(hMnu);
			int mnuStrCnt=0;

			if (mnuCnt!=0)
			{
				//add menu to grammar
				int i;
				command command1;
				StringBuilder mnuStr=new StringBuilder(50);
				ArrayList phraseList=new ArrayList();

				for (i=0;i<mnuCnt;i++)
				{
					//get sting from menu ... to mnuString
					GetMenuString(hMnu,i,mnuStr,50,-1);

					//make sure its not a separator
					if (mnuStr.ToString()!="")
					{
						//save in commnd1.ruleName only the underlined letter
						command1.ruleName=mnuStr.ToString();
						command1.ruleName=command1.ruleName[command1.ruleName.IndexOf('&')+1].ToString();

						//save in command1.phrase the word (without &)
						command1.phrase=mnuStr.ToString();
						if (command1.phrase.IndexOf('&')!=-1)
							command1.phrase=command1.phrase.Remove(command1.phrase.IndexOf('&'),1);
						//also delete every thing after tab ...
						if (command1.phrase.IndexOf('\t')!=-1)
							command1.phrase=command1.phrase.Remove(command1.phrase.IndexOf('\t'),command1.phrase.Length-command1.phrase.IndexOf('\t'));

						//or ...
						if (command1.phrase.IndexOf('.')!=-1)
							command1.phrase=command1.phrase.Remove(command1.phrase.IndexOf('.'),command1.phrase.Length-command1.phrase.IndexOf('.'));

						phraseList.Add(command1);

						//also count actual menu (without separators)
						mnuStrCnt++;
					}
				}

				//add the phraseList (menu) to grammar
				SAPIGrammarFromArrayList(phraseList);

				//notify user
				label1.Text="Menu... "+mnuStrCnt.ToString();
			}
		}

		/// <summary>
		/// hook submenu and add all items to grammar
		/// </summary>
		/// <param name="menuLetter">the letter after "&" that activate the menu</param>
		private void hookSubmenu(string menuLetter)
		{
			//check if there is any menu and hook it
			IntPtr hWnd=GetForegroundWindow();
			IntPtr hMnu=GetMenu(hWnd);
			int mnuCnt=GetMenuItemCount(hMnu);
			StringBuilder mnuStr=new StringBuilder(50);
			IntPtr hSubMnu;
			string str;
			int i=0;

			if (mnuCnt!=0)
			{
				//find menu position
				for (i=0;i<mnuCnt;i++)
				{
					//get sting from menu ... to mnuString
					GetMenuString(hMnu,i,mnuStr,50,-1);
					str=mnuStr.ToString();
					str=str[str.IndexOf('&')+1].ToString();

					//check if this is the menu ...
					if (menuLetter==str)
					{
						//get submenu
						hSubMnu=GetSubMenu(hMnu,i);

						//deal with it as a menu
						hookMenu(hSubMnu);                        

						//item found, we don't need to check other menu items
						break;
					}
				}
			}
		}

		/// <summary>
		/// activate timer for start tracking agent location
		/// </summary>
		private void axAgent1_DragStart(object sender, AxAgentObjects._AgentEvents_DragStartEvent e)
		{
            timer1.Enabled=true;
		}

		/// <summary>
		/// deactivate timer and stop tracking
		/// </summary>
		private void axAgent1_DragComplete(object sender, AxAgentObjects._AgentEvents_DragCompleteEvent e)
		{
			timer1.Enabled=false;
		}

		/// <summary>
		/// locate agent position and update the main form according to it
		/// </summary>
		private void timer1_Tick_1(object sender, System.EventArgs e)
		{
			//reposition frmMain
			this.Left=agent1.Left;
			this.Top=agent1.Top;
		}

		/// <summary>
		/// "Add favirites" pressed from context menu
		/// </summary>
		private void menuItem4_Click(object sender, System.EventArgs e)
		{
			//stop listen (only if it not already)
			if (menuItem3.Text=="Stop listen")
			{
				menuItem3_Click(sender,e);
			}

			frmAddFavorites frm1=new frmAddFavorites();
			// send file name to the form
			frm1.Tag=appPath+"XMLFavorites.xml";
			frm1.Show();
		}

		/// <summary>
		/// "Change character" pressed from context menu
		/// </summary>
		private void menuItem6_Click(object sender, System.EventArgs e)
		{
			if (openFileDialog1.ShowDialog()==DialogResult.OK)
			{
				//Only if agent enabled
				if (menuItem14.Checked==true)
				{
					//unload first character
					axAgent1.Characters.Unload("agentAssistant");
				}

				//loading new character
				try
				{
					axAgent1.Characters.Load("agentAssistant",openFileDialog1.FileName);
					agent1=axAgent1.Characters.Character("agentAssistant");

					//reposition character
					agent1.Left=(short)this.Left;
					agent1.Top=(short)this.Top;

					//show character
					agent1.Show(false);
					agent1.Play("Greet");
					agent1.Play("Explain");
					agent1.Speak("I'm under your control.","");
					agent1.Play("Acknowledge");
					agent1.Speak("just activate me when ever you want...","");
					agent1.Play("Wave");
					agent1.Hide(false);

					//update context menu
					menuItem14.Checked=true;

					//loading initial grammar
					SAPIGrammarFromFile("XMLActivate.xml");

					//notify user
					label1.Text="Deactivate";
					label2.Text="Accuracy 0%";
				}
				catch
				{
					MessageBox.Show("Error loading character\nThe agent will be disabled...","Error - character");
					menuItem14.Checked=false;
				}
			}
		}

		/// <summary>
		/// activating "press key" and simulate press every 0.5 sec
		/// </summary>
		private void timer2_Tick(object sender, System.EventArgs e)
		{
			keybd_event(keyHolding,0,0,0);	//key down
			keybd_event(keyHolding,0,2,0);	//key up
		}

		/// <summary>
		/// load grammar after closing form
		/// must be done here (not in frmAbout and not in recognition event)
		/// because it could be closed by speech or by pressing button
		/// </summary>
		private void frmAbout1_Closed(object sender, System.EventArgs e)
		{
			//load grammar
			SAPIGrammarFromFile("XMLActivate.xml");
			frmAbout1=null;

			//notify user
			label1.Text="Deactivate";
		}

		/// <summary>
		/// load grammar after closing form
		/// must be done here (not in frmCommands and not in recognition event)
		/// because it could be closed by speech or by pressing button
		/// </summary>
		private void frmCommands1_Closed(object sender, System.EventArgs e)
		{
			//load previous grammar
			SAPIGrammarFromFile(previousGrammar);
			frmCommands1=null;
		}

		/// <summary>
		/// load grammar after closing form
		/// must be done here (not in frmFavorites and not in recognition event)
		/// because it could be closed by speech or by pressing button
		/// </summary>
		private void frmFavorites1_Closed(object sender, System.EventArgs e)
		{
			//load grammar
			SAPIGrammarFromFile("XMLDeactivate.xml");
			frmFavorites1=null;

			//notify user
			label1.Text="Activate";
		}

		/// <summary>
		/// show accuracy change form
		/// </summary>
		private void menuItem7_Click(object sender, System.EventArgs e)
		{
			frmAccuracyChange1=new frmAccuracyChange();
			frmAccuracyChange1.Tag=accuracyLimit;
			//also catch the closed event to see result returned
			frmAccuracyChange1.Closed+=new EventHandler(frmAccuracyChange1_Closed);
			frmAccuracyChange1.Show();
		}

		/// <summary>
		/// check the result returned after closing frmAccuracyChange1
		/// if result != -1 change accuracyLimit
		/// </summary>
		private void frmAccuracyChange1_Closed(object sender, System.EventArgs e)
		{
			if ((int)frmAccuracyChange1.Tag!=-1)
				accuracyLimit=(int)frmAccuracyChange1.Tag;
		}

		/// <summary>
		/// User training wizard
		/// </summary>
		private void menuItem8_Click(object sender, System.EventArgs e)
		{
			object str1="";

			if (objRecoContext.Recognizer.IsUISupported("UserTraining",ref str1)==true)
			{
				//stop listen (only if it not already)
				if (menuItem3.Text=="Stop listen")
				{
					menuItem3_Click(sender,e);
				}

				objRecoContext.Recognizer.DisplayUI((int)this.Handle,"SR","UserTraining",ref str1);
			}
			else
				MessageBox.Show("User training wizard not supported !","ERROR - activating wizard");
		}

		/// <summary>
		/// Mic training wizard
		/// </summary>
		private void menuItem9_Click(object sender, System.EventArgs e)
		{
			object str1="";

			if (objRecoContext.Recognizer.IsUISupported("MicTraining",ref str1)==true)
			{
				//stop listen (only if it not already)
				if (menuItem3.Text=="Stop listen")
				{
					menuItem3_Click(sender,e);
				}

				objRecoContext.Recognizer.DisplayUI((int)this.Handle,"SR","MicTraining",ref str1);
			}
			else
				MessageBox.Show("Mic training wizard not supported !","ERROR - activating wizard");
		}

		/// <summary>
		/// Change user profile
		/// </summary>
		private void menuItem10_Click(object sender, System.EventArgs e)
		{
			//reset
			initSAPI();

			frmProfileChange1=new frmProfileChange();
			frmProfileChange1.Closed+=new EventHandler(frmProfileChange1_Closed);
			//send objRecoContext
			frmProfileChange1.Tag=objRecoContext;
			frmProfileChange1.Show();
		}

		/// <summary>
		/// only after closing form we can laod grammar again
		/// </summary>
		private void frmProfileChange1_Closed(object sender, System.EventArgs e)
		{
			//loading initial grammar
			SAPIGrammarFromFile("XMLActivate.xml");

			//notify user
			label1.Text="Deactivate";
			label2.Text="Accuracy 0%";
		}

		/// <summary>
		/// Profile proporties wizard
		/// </summary>
		private void menuItem13_Click(object sender, System.EventArgs e)
		{
			object str1="";

			if (objRecoContext.Recognizer.IsUISupported("RecoProfileProperties",ref str1)==true)
			{
				//stop listen (only if it not already)
				if (menuItem3.Text=="Stop listen")
				{
					menuItem3_Click(sender,e);
				}

				objRecoContext.Recognizer.DisplayUI((int)this.Handle,"SR","RecoProfileProperties",ref str1);
			}
			else
				MessageBox.Show("Profile Properties wizard not supported !","ERROR - activating wizard");
		}

		private void menuItem14_Click(object sender, System.EventArgs e)
		{
			if (menuItem14.Checked==true)
			{
				menuItem14.Checked=false;
				agent1.StopAll("");
				agent1.Hide(false);
			}
			else
			{
				if (agent1!=null)
				{
					menuItem14.Checked=true;
					agent1.Show(false);
				}
				else
					MessageBox.Show("There is no agent loaded...\nYou will need to load agent first.","Error - character");
			}
		}
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Tambi Ashmoz

Israel Israel
No Biography provided

| Advertise | Privacy | Terms of Use | Mobile
Web03 | 2.8.141216.1 | Last Updated 27 Feb 2004
Article Copyright 2004 by Tambi Ashmoz
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid