Attachment #100368 for bug #152652




    for example
    \begin{verbatim}
       EST_DiscreteProbistribution pdf;
       for (long i=pdf.item_start(); i < pdf.item_end(); i=pdf.item_next(i))
       {
          EST_String name;
          double prob;

    /// 
    double frequency(const int i) const; 
    /// Used for iterating through members of the distribution
    long item_start() const;
    /// Used for iterating through members of the distribution
    long item_next(long idx) const;
    /// Used for iterating through members of the distribution
    int item_end(long idx) const;
    /// During iteration returns name given index 
    const EST_String &item_name(long idx) const;
    /// During iteration returns name and frequency given index  
    void item_freq(long idx,EST_String &s,double &freq) const;
    /// During iteration returns name and probability given index
    void item_prob(long idx,EST_String &s,double &prob) const;

    /// Returns discrete vocabulary of distribution
    inline const EST_Discrete *const get_discrete() const { return discrete; };

Lines 537-543 ostream & operator <<(ostream &s, WImpur Link Here

(-)speech_tools/stats/wagon/wagon_aux.cc (-1 / +1 lines)
537	}	537	}
538	else if (imp.t == wnim_class)	538	else if (imp.t == wnim_class)
539	{	539	{
540	int i;	540	long i;
541	EST_String name;	541	EST_String name;
542	double prob;	542	double prob;
543		543




}

//  For iterating through members of a probability distribution
long EST_DiscreteProbDistribution::item_start(void) const
{
    if (type == tprob_discrete)
	return 0;
    else
	return (long)scounts.list.head();
}

int EST_DiscreteProbDistribution::item_end(long idx) const
{
    if (type == tprob_discrete)
	return (idx >= icounts.length());

	return ((EST_Litem *)idx == 0);
}

long EST_DiscreteProbDistribution::item_next(long idx) const
{
    if (type == tprob_discrete)
	return ++idx;
    else
	return (long)next((EST_Litem *)idx);
}

const EST_String &EST_DiscreteProbDistribution::item_name(long idx) const
{
    if (type == tprob_discrete)
	return discrete->name(idx);

	return scounts.list((EST_Litem *)idx).k;
}

void EST_DiscreteProbDistribution::item_freq(long idx,EST_String &s,double &freq) const
{
    if (type == tprob_discrete)
    {

    }
}

void EST_DiscreteProbDistribution::item_prob(long idx,EST_String &s,double &prob) const
{
    if (type == tprob_discrete)
    {

ostream & operator<<(ostream &s, const EST_DiscreteProbDistribution &pd)
{
    // Output best with probabilities
    long i;
    double prob;
    double sum=0;
    EST_String name;




					  const double count)
{

//    long i;
//    cerr << "accumulate level " << p_level << " : ";
//    for(i=0;i<words.n();i++)
//    {

    // not right - just print out, then recurse through children
    // change to use 'backoff_traverse'
    
    long k;
    double freq;
    EST_String name;
    for (k=p_pdf.item_start();

{

    // recursively delete this state and all its children
    long k;
    double freq;
    EST_String name;
    for (k=p_pdf.item_start();


void EST_BackoffNgrammarState::frequency_of_frequencies(EST_DVector &ff)
{
    long k; int max=ff.n();
    double freq;
    EST_String name;
    for (k=p_pdf.item_start();

{
    
    /*
       long i;
       for(i=0;i<words.n();i++)
       {
       cerr << vocab_pdf.item_name(words(i));

    // remove any branches with zero frequency count
    
    // find children of this state with zero freq and zap them
    long k;
    double freq;
    EST_String name;
    for (k=start_state->pdf_const().item_start();

	backoff_representation->print_freqs(os,p_order);
    else
    {
	int i,j; long k;
	EST_IVector window(p_order-1);
	
	for (i=0; i < p_num_states; i++)

    function(start_state,params);
    
    // and recurse down the tree
    long k;
    double freq;
    EST_String name;
    for (k=start_state->pdf_const().item_start();

    {
	// and recurse down the tree if we haven't
	// reached the level yet
	long k;
	double freq;
	EST_String name;
	




load_ngram_cstr_bin(const EST_String filename, EST_Ngrammar &n)
{
    EST_TokenStream ts;
    int i,j,order; long k;
    int num_entries;
    double approx_num_samples = 0.0;
    long freq_data_start, freq_data_end;

save_ngram_htk_ascii_sub(const EST_String &word, ostream *ost, 
			 EST_Ngrammar &n, double floor)
{
    long k;
    EST_String name;
    double freq;
    EST_StrVector this_ngram(2); // assumes bigram

    // awb's format
    (void)trace;
    ostream *ost;
    int i; long k;
    
    if (filename == "-")
	ost = &cout;

    if (n.representation() == EST_Ngrammar::sparse)
	return misc_write_error;
    
    int i; long k;
    FILE *ofd;
    double lfreq = -1;
    double count = -1;





void make_f_of_f(EST_BackoffNgrammarState *s,void *params)
{
    long k;
    double freq;
    EST_String name;



void get_max_f(EST_BackoffNgrammarState *s,void *params)
{
    long k;
    double freq;
    EST_String name;



void map_f_of_f(EST_BackoffNgrammarState *s,void *params)
{
    long k;
    double freq;
    EST_String name;



void zero_small_f(EST_BackoffNgrammarState *s,void *params)
{
    long k;
    double freq;
    EST_String name;



void frequency_of_frequencies(EST_DVector &ff, EST_Ngrammar &n,int this_order)
{
  int i,size; long k;
  double max=0.0;

  // if ff has zero size, do complete frequency of frequencies


void map_frequencies(EST_Ngrammar &n, const EST_DVector &map, const int this_order)
{
  int i; long k;


  switch(n.representation())

Lines 71-77 EST_PredictionSuffixTree_tree_node::prin Link Here

(-)speech_tools/grammar/ngram/EST_PST.cc (-2 / +2 lines)
71	// Base -- print from pd	71	// Base -- print from pd
72	EST_String s;	72	EST_String s;
73	double freq;	73	double freq;
74	for (int i = pd.item_start();	74	for (long i = pd.item_start();
75	!pd.item_end(i);	75	!pd.item_end(i);
76	i=pd.item_next(i))	76	i=pd.item_next(i))
77	{	77	{
Lines 98-104 EST_PredictionSuffixTree_tree_node::prin Link Here
98	EST_String s;	98	EST_String s;
99	double prob;	99	double prob;
100	os << get_path() << " :";	100	os << get_path() << " :";
101	for (int i = pd.item_start(); !pd.item_end(i) ; i=pd.item_next(i))	101	for (long i = pd.item_start(); !pd.item_end(i) ; i=pd.item_next(i))
102	{	102	{
103	pd.item_prob(i,s,prob);	103	pd.item_prob(i,s,prob);
104	os << " " << s << " " << prob;	104	os << " " << s << " " << prob;

Lines 74-80 void fs_build_backoff_ngrams(EST_Ngramma Link Here

(-)speech_tools/grammar/ngram/freqsmooth.cc (-2 / +2 lines)
74	EST_Ngrammar &ngram)	74	EST_Ngrammar &ngram)
75	{	75	{
76	// Build all the backoff grammars back to uni-grams	76	// Build all the backoff grammars back to uni-grams
77	int i,j,k,l;	77	int i,j,l; long k;
78		78
79	for (i=0; i < ngram.order()-1; i++)	79	for (i=0; i < ngram.order()-1; i++)
80	backoff_ngrams[i].init(i+1,EST_Ngrammar::dense,	80	backoff_ngrams[i].init(i+1,EST_Ngrammar::dense,
Lines 110-116 int fs_backoff_smooth(EST_Ngrammar *back Link Here
110	{	110	{
111	// For all ngrams which are too infrequent, adjust their	111	// For all ngrams which are too infrequent, adjust their
112	// frequencies based on their backoff probabilities	112	// frequencies based on their backoff probabilities
113	int i,j;	113	int i; long j;
114	double occurs;	114	double occurs;
115	double backoff_prob;	115	double backoff_prob;
116		116




    LISP *ssplits;
    gc_protect(&splits);
    EST_String sname;
    int b,best_b; long i;
    int num_pdfs;
    double best_score, score, sfreq;


    // Find score of (a+b) vs (all-(a+b))
    EST_DiscreteProbDistribution ab(a);
    EST_DiscreteProbDistribution all_but_ab(all);
    long i;
    EST_String sname;
    double sfreq, score;
    for (i=b.item_start(); !b.item_end(i);

    EST_DiscreteProbDistribution pdf_split(&wfst.in_symbols());
    EST_DiscreteProbDistribution pdf_remain(&wfst.in_symbols());
    int in, tostate, id;
    long i;
    double sfreq;
    EST_String sname;


Return to bug 152652

Lines 194-200 enum EST_tprob_type {tprob_string, tprob Link Here

(-)speech_tools/include/EST_simplestats.h (-7 / +7 lines)
194	for example	194	for example
195	\begin{verbatim}	195	\begin{verbatim}
196	EST_DiscreteProbistribution pdf;	196	EST_DiscreteProbistribution pdf;
197	for (int i=pdf.item_start(); i < pdf.item_end(); i=pdf.item_next(i))	197	for (long i=pdf.item_start(); i < pdf.item_end(); i=pdf.item_next(i))
198	{	198	{
199	EST_String name;	199	EST_String name;
200	double prob;	200	double prob;
Lines 265-281 public: Link Here
265	///	265	///
266	double frequency(const int i) const;	266	double frequency(const int i) const;
267	/// Used for iterating through members of the distribution	267	/// Used for iterating through members of the distribution
268	int item_start() const;	268	long item_start() const;
269	/// Used for iterating through members of the distribution	269	/// Used for iterating through members of the distribution
270	int item_next(int idx) const;	270	long item_next(long idx) const;
271	/// Used for iterating through members of the distribution	271	/// Used for iterating through members of the distribution
272	int item_end(int idx) const;	272	int item_end(long idx) const;
273	/// During iteration returns name given index	273	/// During iteration returns name given index
274	const EST_String &item_name(int idx) const;	274	const EST_String &item_name(long idx) const;
275	/// During iteration returns name and frequency given index	275	/// During iteration returns name and frequency given index
276	void item_freq(int idx,EST_String &s,double &freq) const;	276	void item_freq(long idx,EST_String &s,double &freq) const;
277	/// During iteration returns name and probability given index	277	/// During iteration returns name and probability given index
278	void item_prob(int idx,EST_String &s,double &prob) const;	278	void item_prob(long idx,EST_String &s,double &prob) const;
279		279
280	/// Returns discrete vocabulary of distribution	280	/// Returns discrete vocabulary of distribution
281	inline const EST_Discrete *const get_discrete() const { return discrete; };	281	inline const EST_Discrete *const get_discrete() const { return discrete; };

Lines 305-319 double EST_DiscreteProbDistribution::ent Link Here

(-)speech_tools/stats/EST_DProbDist.cc (-9 / +9 lines)
305	}	305	}
306		306
307	// For iterating through members of a probability distribution	307	// For iterating through members of a probability distribution
308	int EST_DiscreteProbDistribution::item_start(void) const	308	long EST_DiscreteProbDistribution::item_start(void) const
309	{	309	{
310	if (type == tprob_discrete)	310	if (type == tprob_discrete)
311	return 0;	311	return 0;
312	else	312	else
313	return (int)scounts.list.head();	313	return (long)scounts.list.head();
314	}	314	}
315		315
316	int EST_DiscreteProbDistribution::item_end(int idx) const	316	int EST_DiscreteProbDistribution::item_end(long idx) const
317	{	317	{
318	if (type == tprob_discrete)	318	if (type == tprob_discrete)
319	return (idx >= icounts.length());	319	return (idx >= icounts.length());
Lines 321-335 int EST_DiscreteProbDistribution::item_e Link Here
321	return ((EST_Litem *)idx == 0);	321	return ((EST_Litem *)idx == 0);
322	}	322	}
323		323
324	int EST_DiscreteProbDistribution::item_next(int idx) const	324	long EST_DiscreteProbDistribution::item_next(long idx) const
325	{	325	{
326	if (type == tprob_discrete)	326	if (type == tprob_discrete)
327	return ++idx;	327	return ++idx;
328	else	328	else
329	return (int)next((EST_Litem *)idx);	329	return (long)next((EST_Litem *)idx);
330	}	330	}
331		331
332	const EST_String &EST_DiscreteProbDistribution::item_name(int idx) const	332	const EST_String &EST_DiscreteProbDistribution::item_name(long idx) const
333	{	333	{
334	if (type == tprob_discrete)	334	if (type == tprob_discrete)
335	return discrete->name(idx);	335	return discrete->name(idx);
Lines 337-343 const EST_String &EST_DiscreteProbDistri Link Here
337	return scounts.list((EST_Litem *)idx).k;	337	return scounts.list((EST_Litem *)idx).k;
338	}	338	}
339		339
340	void EST_DiscreteProbDistribution::item_freq(int idx,EST_String &s,double &freq) const	340	void EST_DiscreteProbDistribution::item_freq(long idx,EST_String &s,double &freq) const
341	{	341	{
342	if (type == tprob_discrete)	342	if (type == tprob_discrete)
343	{	343	{
Lines 351-357 void EST_DiscreteProbDistribution::item_ Link Here
351	}	351	}
352	}	352	}
353		353
354	void EST_DiscreteProbDistribution::item_prob(int idx,EST_String &s,double &prob) const	354	void EST_DiscreteProbDistribution::item_prob(long idx,EST_String &s,double &prob) const
355	{	355	{
356	if (type == tprob_discrete)	356	if (type == tprob_discrete)
357	{	357	{
Lines 368-374 void EST_DiscreteProbDistribution::item_ Link Here
368	ostream & operator<<(ostream &s, const EST_DiscreteProbDistribution &pd)	368	ostream & operator<<(ostream &s, const EST_DiscreteProbDistribution &pd)
369	{	369	{
370	// Output best with probabilities	370	// Output best with probabilities
371	int i;	371	long i;
372	double prob;	372	double prob;
373	double sum=0;	373	double sum=0;
374	EST_String name;	374	EST_String name;

Lines 180-186 bool EST_BackoffNgrammarState::accumulat Link Here

(-)speech_tools/grammar/ngram/EST_Ngrammar.cc (-9 / +9 lines)
180	const double count)	180	const double count)
181	{	181	{
182		182
183	// int i;	183	// long i;
184	// cerr << "accumulate level " << p_level << " : ";	184	// cerr << "accumulate level " << p_level << " : ";
185	// for(i=0;i<words.n();i++)	185	// for(i=0;i<words.n();i++)
186	// {	186	// {
Lines 302-308 void EST_BackoffNgrammarState::print_fre Link Here
302	// not right - just print out, then recurse through children	302	// not right - just print out, then recurse through children
303	// change to use 'backoff_traverse'	303	// change to use 'backoff_traverse'
304		304
305	int k;	305	long k;
306	double freq;	306	double freq;
307	EST_String name;	307	EST_String name;
308	for (k=p_pdf.item_start();	308	for (k=p_pdf.item_start();
Lines 369-375 void EST_BackoffNgrammarState::zap() Link Here
369	{	369	{
370		370
371	// recursively delete this state and all its children	371	// recursively delete this state and all its children
372	int k;	372	long k;
373	double freq;	373	double freq;
374	EST_String name;	374	EST_String name;
375	for (k=p_pdf.item_start();	375	for (k=p_pdf.item_start();
Lines 452-458 bool EST_BackoffNgrammarState::set_backo Link Here
452		452
453	void EST_BackoffNgrammarState::frequency_of_frequencies(EST_DVector &ff)	453	void EST_BackoffNgrammarState::frequency_of_frequencies(EST_DVector &ff)
454	{	454	{
455	int k,max=ff.n();	455	long k; int max=ff.n();
456	double freq;	456	double freq;
457	EST_String name;	457	EST_String name;
458	for (k=p_pdf.item_start();	458	for (k=p_pdf.item_start();
Lines 911-917 void EST_Ngrammar::accumulate(const EST_ Link Here
911	{	911	{
912		912
913	/*	913	/*
914	int i;	914	long i;
915	for(i=0;i<words.n();i++)	915	for(i=0;i<words.n();i++)
916	{	916	{
917	cerr << vocab_pdf.item_name(words(i));	917	cerr << vocab_pdf.item_name(words(i));
Lines 1581-1587 void EST_Ngrammar::prune_backoff_represe Link Here
1581	// remove any branches with zero frequency count	1581	// remove any branches with zero frequency count
1582		1582
1583	// find children of this state with zero freq and zap them	1583	// find children of this state with zero freq and zap them
1584	int k;	1584	long k;
1585	double freq;	1585	double freq;
1586	EST_String name;	1586	EST_String name;
1587	for (k=start_state->pdf_const().item_start();	1587	for (k=start_state->pdf_const().item_start();
Lines 2320-2326 void EST_Ngrammar::print_freqs(ostream & Link Here
2320	backoff_representation->print_freqs(os,p_order);	2320	backoff_representation->print_freqs(os,p_order);
2321	else	2321	else
2322	{	2322	{
2323	int i,j,k;	2323	int i,j; long k;
2324	EST_IVector window(p_order-1);	2324	EST_IVector window(p_order-1);
2325		2325
2326	for (i=0; i < p_num_states; i++)	2326	for (i=0; i < p_num_states; i++)
Lines 2661-2667 EST_Ngrammar::backoff_traverse(EST_Backo Link Here
2661	function(start_state,params);	2661	function(start_state,params);
2662		2662
2663	// and recurse down the tree	2663	// and recurse down the tree
2664	int k;	2664	long k;
2665	double freq;	2665	double freq;
2666	EST_String name;	2666	EST_String name;
2667	for (k=start_state->pdf_const().item_start();	2667	for (k=start_state->pdf_const().item_start();
Lines 2692-2698 EST_Ngrammar::backoff_traverse(EST_Backo Link Here
2692	{	2692	{
2693	// and recurse down the tree if we haven't	2693	// and recurse down the tree if we haven't
2694	// reached the level yet	2694	// reached the level yet
2695	int k;	2695	long k;
2696	double freq;	2696	double freq;
2697	EST_String name;	2697	EST_String name;
2698		2698

Lines 281-287 EST_read_status Link Here

(-)speech_tools/grammar/ngram/ngrammar_io.cc (-4 / +4 lines)
281	load_ngram_cstr_bin(const EST_String filename, EST_Ngrammar &n)	281	load_ngram_cstr_bin(const EST_String filename, EST_Ngrammar &n)
282	{	282	{
283	EST_TokenStream ts;	283	EST_TokenStream ts;
284	int i,j,k,order;	284	int i,j,order; long k;
285	int num_entries;	285	int num_entries;
286	double approx_num_samples = 0.0;	286	double approx_num_samples = 0.0;
287	long freq_data_start, freq_data_end;	287	long freq_data_start, freq_data_end;
Lines 407-413 EST_write_status Link Here
407	save_ngram_htk_ascii_sub(const EST_String &word, ostream *ost,	407	save_ngram_htk_ascii_sub(const EST_String &word, ostream *ost,
408	EST_Ngrammar &n, double floor)	408	EST_Ngrammar &n, double floor)
409	{	409	{
410	int k;	410	long k;
411	EST_String name;	411	EST_String name;
412	double freq;	412	double freq;
413	EST_StrVector this_ngram(2); // assumes bigram	413	EST_StrVector this_ngram(2); // assumes bigram
Lines 734-740 save_ngram_cstr_ascii(const EST_String f Link Here
734	// awb's format	734	// awb's format
735	(void)trace;	735	(void)trace;
736	ostream *ost;	736	ostream *ost;
737	int i,k;	737	int i; long k;
738		738
739	if (filename == "-")	739	if (filename == "-")
740	ost = &cout;	740	ost = &cout;
Lines 831-837 save_ngram_cstr_bin(const EST_String fil Link Here
831	if (n.representation() == EST_Ngrammar::sparse)	831	if (n.representation() == EST_Ngrammar::sparse)
832	return misc_write_error;	832	return misc_write_error;
833		833
834	int i,k;	834	int i; long k;
835	FILE *ofd;	835	FILE *ofd;
836	double lfreq = -1;	836	double lfreq = -1;
837	double count = -1;	837	double count = -1;

Lines 117-123 smooth_ExponentialFit(EST_DVector &N, in Link Here

(-)speech_tools/grammar/ngram/ngrammar_aux.cc (-6 / +6 lines)
117		117
118	void make_f_of_f(EST_BackoffNgrammarState s,void params)	118	void make_f_of_f(EST_BackoffNgrammarState s,void params)
119	{	119	{
120	int k;	120	long k;
121	double freq;	121	double freq;
122	EST_String name;	122	EST_String name;
123		123
Lines 138-144 void make_f_of_f(EST_BackoffNgrammarStat Link Here
138		138
139	void get_max_f(EST_BackoffNgrammarState s,void params)	139	void get_max_f(EST_BackoffNgrammarState s,void params)
140	{	140	{
141	int k;	141	long k;
142	double freq;	142	double freq;
143	EST_String name;	143	EST_String name;
144		144
Lines 158-164 void get_max_f(EST_BackoffNgrammarState Link Here
158		158
159	void map_f_of_f(EST_BackoffNgrammarState s,void params)	159	void map_f_of_f(EST_BackoffNgrammarState s,void params)
160	{	160	{
161	int k;	161	long k;
162	double freq;	162	double freq;
163	EST_String name;	163	EST_String name;
164		164
Lines 184-190 void map_f_of_f(EST_BackoffNgrammarState Link Here
184		184
185	void zero_small_f(EST_BackoffNgrammarState s,void params)	185	void zero_small_f(EST_BackoffNgrammarState s,void params)
186	{	186	{
187	int k;	187	long k;
188	double freq;	188	double freq;
189	EST_String name;	189	EST_String name;
190		190
Lines 204-210 void zero_small_f(EST_BackoffNgrammarSta Link Here
204		204
205	void frequency_of_frequencies(EST_DVector &ff, EST_Ngrammar &n,int this_order)	205	void frequency_of_frequencies(EST_DVector &ff, EST_Ngrammar &n,int this_order)
206	{	206	{
207	int i,k,size;	207	int i,size; long k;
208	double max=0.0;	208	double max=0.0;
209		209
210	// if ff has zero size, do complete frequency of frequencies	210	// if ff has zero size, do complete frequency of frequencies
Lines 302-308 void frequency_of_frequencies(EST_DVecto Link Here
302		302
303	void map_frequencies(EST_Ngrammar &n, const EST_DVector &map, const int this_order)	303	void map_frequencies(EST_Ngrammar &n, const EST_DVector &map, const int this_order)
304	{	304	{
305	int i,k;	305	int i; long k;
306		306
307		307
308	switch(n.representation())	308	switch(n.representation())

Lines 299-305 static LISP find_best_split(EST_WFST &wf Link Here

(-)speech_tools/grammar/wfst/wfst_train.cc (-3 / +3 lines)
299	LISP *ssplits;	299	LISP *ssplits;
300	gc_protect(&splits);	300	gc_protect(&splits);
301	EST_String sname;	301	EST_String sname;
302	int b,best_b,i;	302	int b,best_b; long i;
303	int num_pdfs;	303	int num_pdfs;
304	double best_score, score, sfreq;	304	double best_score, score, sfreq;
305		305
Lines 374-380 static double score_pdf_combine(EST_Disc Link Here
374	// Find score of (a+b) vs (all-(a+b))	374	// Find score of (a+b) vs (all-(a+b))
375	EST_DiscreteProbDistribution ab(a);	375	EST_DiscreteProbDistribution ab(a);
376	EST_DiscreteProbDistribution all_but_ab(all);	376	EST_DiscreteProbDistribution all_but_ab(all);
377	int i;	377	long i;
378	EST_String sname;	378	EST_String sname;
379	double sfreq, score;	379	double sfreq, score;
380	for (i=b.item_start(); !b.item_end(i);	380	for (i=b.item_start(); !b.item_end(i);
Lines 506-512 static double find_score_if_split(EST_WF Link Here
506	EST_DiscreteProbDistribution pdf_split(&wfst.in_symbols());	506	EST_DiscreteProbDistribution pdf_split(&wfst.in_symbols());
507	EST_DiscreteProbDistribution pdf_remain(&wfst.in_symbols());	507	EST_DiscreteProbDistribution pdf_remain(&wfst.in_symbols());
508	int in, tostate, id;	508	int in, tostate, id;
509	int i;	509	long i;
510	double sfreq;	510	double sfreq;
511	EST_String sname;	511	EST_String sname;
512		512