#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (C) 2011 Adam Radziszewski.
# This program is free software; you can redistribute and/or modify it
# under the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.
#
# See the LICENCE and COPYING files for more details
# CHANGES
# - July 2017 - added statistics for lemma accuracy for known and unknown
#               words separately (Łukasz Kobyliński)

from optparse import OptionParser
import sys
import corpus2
from StringIO import StringIO
from collections import defaultdict as dd
descr = """%prog [options] TAGDFOLD1 ... REFFOLD1 ...

Evaluates tagging of tagged corpus consisting of possibly several folds using
the given reference corpus (gold standard). The subsequent folds should match.

E.g. PYTHONIOENCODING=utf8 ./tagger-eval.py tagged??.xml folds/test??.xml | tee results.txt

"""

changelog = """
* respect_spaces
* report POS hits also
* comments
* fix debug strings
* skip non-disamb tokens as Pantera artefacts
* higher frac precision in output
* extract measures to functions for averaging
* averaging over folds
* separate stats for unknown forms
* evaluate lemmatisation lower bound (weak and strong)
* get rid of heuristic punctuation tweaks and WC measure (without bounds)
* case-insensitive lemma comparison (strong lemma nocase lower bound)
* case-sensitive and case-insensitive concatenative heuristic for upper bound
  (if concatenation of all segments in a fragment gives the same text)
* option to process only first lexeme of each token (by default off)
"""

def text(tok_seq, respect_spaces, mark_boundaries = False):
	"""Extracts text from a sequence of tokens. If respect_spaces, will append
	spaces between tokens where no no-space markers present."""
	buff = StringIO()
	nonfirst = False
	for tok in tok_seq:
		if nonfirst and respect_spaces and tok.after_space():
			buff.write(' ')
		if mark_boundaries:
			buff.write('[')
		buff.write(tok.orth_utf8().decode('utf-8'))
		if mark_boundaries:
			buff.write(']')
		nonfirst = True
	return buff.getvalue()

def next_tok(rdr):
	while True:
		tok = rdr.get_next_token()
		if not tok:
			return tok
		if tok.has_disamb_lexeme():
			return tok

def tok_seqs(rdr_here, rdr_there, respect_spaces, verbose_mode, debug_mode):
	"""Generates subsequent aligned token sequences from the two readers.
	Alignment is performed on the text level. Shortest token sequences with
	matching text are generated.
	"""
	
	tok_here = next_tok(rdr_here)
	tok_there = next_tok(rdr_there)
	
	assert tok_here and tok_there, 'no input'
	
	buff_here = [tok_here]
	buff_there = [tok_there]
	
	LIMIT = 30
	
	num_iter = 0
	
	while True:
		num_iter += 1
		if num_iter % 10000 == 0: print num_iter, 'iterations...'
		if len(buff_here) > LIMIT or len(buff_there) > LIMIT:
			raise IOError('limit exceeded')
		
		text_here = text(buff_here, respect_spaces)
		text_there = text(buff_there, respect_spaces)
		
		if debug_mode:
			print '%s (%d) v. %s(%d)' % (text_here, len(text_here),
			text_there, len(text_there))
		
		if len(text_here) == len(text_there):
			# already aligned
			if text_here != text_there:
				# the same text length but different chars
				# won't recover, flush it as it is
				print 'WARNING: mismatch', text_here, '|', text_there
			yield (buff_here, buff_there)
			buff_here = []
			buff_there = []
			tok_here = next_tok(rdr_here)
			tok_there = next_tok(rdr_there)
			if not tok_here or not tok_there:
				assert not tok_here and not tok_there, 'one ended prematurely'
				break
			buff_here = [tok_here]
			buff_there = [tok_there]
		elif len(text_here) < len(text_there):
			# try fetching here
			tok_here = next_tok(rdr_here)
			assert tok_here, 'first ended prematurely'
			buff_here.append(tok_here)
		else: # len(text_here) > len(text_there):
			# try fetching there
			tok_there = next_tok(rdr_there)
			assert tok_there, 'second ended prematurely'
			buff_there.append(tok_there)
	
	if buff_here or buff_there:
		assert buff_here and buff_there
		yield (buff_here, buff_there)

class Feat:
	WEAK_LEM_HIT = 'weak lem hit' # also includes strong hits
	STRONG_LEM_HIT = 'strong lem hit'
	STRONG_LEM_NOCASE_HIT = 'strong lem nocase hit' # case-insens as above
	WEAK_TAG_HIT = 'weak tag hit' # also includes strong hits
	STRONG_TAG_HIT = 'strong tag hit'
	WEAK_POS_HIT = 'weak pos hit' # also includes strong hits
	STRONG_POS_HIT = 'strong pos hit'
	SEG_NOCHANGE = 'segmentation unchanged'
	SEG_CHANGE = 'segmentation change'
	KNOWN = 'known'
	UNKNOWN = 'unknown'

class Metric:
	# lower bounds for lemmatisation
	WL_LOWER = ([Feat.WEAK_LEM_HIT, Feat.SEG_NOCHANGE], None) # lower bound for weak lemmatisation
	SL_LOWER = ([Feat.STRONG_LEM_HIT, Feat.SEG_NOCHANGE], None) # lower bound for strong lemmatisation
	# separate stats for known and unknown forms
        KN_SL_LOWER = ([Feat.STRONG_LEM_HIT, Feat.SEG_NOCHANGE, Feat.KNOWN], [Feat.KNOWN])
        UNK_SL_LOWER = ([Feat.STRONG_LEM_HIT, Feat.SEG_NOCHANGE, Feat.UNKNOWN], [Feat.UNKNOWN])
	SL_NOCASE_LOWER = ([Feat.STRONG_LEM_NOCASE_HIT, Feat.SEG_NOCHANGE], None) # as above but case-insensitive
	SL_NOCASE_CAT_HEUR = ([Feat.STRONG_LEM_NOCASE_HIT], None) # as above but concatenate fragments and judge if they are equal
	SL_CASE_CAT_HEUR = ([Feat.STRONG_LEM_HIT], None) # as above but case-sensitive
	
	# SL_HUPPER = ([Feat.STRONG_LEM_HIT], None) TODO: -> lemma
	# lower bounds for correctness, treating all segchanges as failures
	WC_LOWER = ([Feat.WEAK_TAG_HIT, Feat.SEG_NOCHANGE], None) # lower bound for weak correctness
	SC_LOWER = ([Feat.STRONG_TAG_HIT, Feat.SEG_NOCHANGE], None) # lower bound for strong correctness
	KN_SC_LOWER = ([Feat.STRONG_TAG_HIT, Feat.SEG_NOCHANGE, Feat.KNOWN], [Feat.KNOWN])
        UNK_SC_LOWER = ([Feat.STRONG_TAG_HIT, Feat.SEG_NOCHANGE, Feat.UNKNOWN], [Feat.UNKNOWN])
	# as above but metric for POS hits
	POS_WC_LOWER = ([Feat.WEAK_POS_HIT, Feat.SEG_NOCHANGE], None) # lower bound for POS WC
	POS_SC_LOWER = ([Feat.STRONG_POS_HIT, Feat.SEG_NOCHANGE], None) # lower bound for POS SC
	# separate stats for known and unknown forms
	KN_WC_LOWER = ([Feat.WEAK_TAG_HIT, Feat.SEG_NOCHANGE, Feat.KNOWN], [Feat.KNOWN])
	UNK_WC_LOWER = ([Feat.WEAK_TAG_HIT, Feat.SEG_NOCHANGE, Feat.UNKNOWN], [Feat.UNKNOWN])
	KN_SEG_CHANGE = ([Feat.SEG_CHANGE, Feat.KNOWN], [Feat.KNOWN])
	UNK_SEG_CHANGE = ([Feat.SEG_CHANGE, Feat.UNKNOWN], [Feat.UNKNOWN])

	KN_POS_SC_LOWER = ([Feat.STRONG_POS_HIT, Feat.SEG_NOCHANGE, Feat.KNOWN], [Feat.KNOWN])
	UNK_POS_SC_LOWER = ([Feat.STRONG_POS_HIT, Feat.SEG_NOCHANGE, Feat.UNKNOWN], [Feat.UNKNOWN])
	# percentage of known and unknown tokens
	KN = ([Feat.KNOWN], None)
	UNK = ([Feat.UNKNOWN], None)
	# percentage of tokens subjected to seg change
	SEG_CHANGE = ([Feat.SEG_CHANGE], None)
	SEG_NOCHANGE = ([Feat.SEG_NOCHANGE], None)

class TokComp:
	"""Creates a tagger evaluation comparator. The comparator reads two
	annotated texts: tagged text (output of a tagger being evaluated) and
	reference tagging (the gold standard). Most of the figures reported concern
	whether a reference token has got some sort of coverage in the tagger
	output.
	
	Differences in segmentation are treated by aligning tokens textually,
	i.e. shortest matching sequences are extracted and then compared.
	
	unk_tag is a string representation of tag used for unknown words.
	Set expand_optional to True if ommission of optional attribute
	values should be treated as multiple tags, each with a different
	variant of the value."""
	def __init__(self, tagset, unk_tag,
		expand_optional, first_lex_only, debug = False):
		self.tagset = tagset
		self.unk_tag = unk_tag
		self.expand_optional = expand_optional
		self.first_lex_only = first_lex_only
		self.debug = debug
		self.ref_toks = 0 # all tokens in ref corpus
		self.tag_toks = 0 # all tokens in tagger output
		self.tag_feat = dd(int) # feat frozenset -> count
	
	def eat_ref_toks(self, feat_set, num_toks):
		"""Classifies num_toks reference tokens as having the given set of
		features. Will also increment the ref_toks counter."""
		self.ref_toks += num_toks
		self.tag_feat[frozenset(feat_set)] += num_toks
	
	def is_unk(self, tok):
		"""There is an 'unknown word' interpretation."""
		tok_tags = [self.tagset.tag_to_string(lex.tag()) for lex in tok.lexemes()]
		return self.unk_tag in tok_tags
	
	def tagstrings_of_token(self, tok):
		"""Returns a set of strings, corresponding to disamb tags
		found in the token. If expand_optional, multiple tags may be
		produced from single-disamb tokens when some optional attributes
		are unspecified."""
		tags = [lex.tag() for lex in tok.lexemes() if lex.is_disamb()]
		assert tags
		if self.first_lex_only:
			# forcing first disamb lexeme only
			tags = tags[:1]
		if self.expand_optional:
			# create multivalue tag wherever a value of optional attr is unspec
			tags = [self.tagset.expand_optional_attrs(tag) for tag in tags]
			# now expand multivalue tags to singular tags
			newtags = []
			for tag in tags:
				newtags.extend(self.tagset.split_tag(tag))
			tags = newtags
			if self.debug:
				print ' + '.join(self.tagset.tag_to_string(tag) for tag in tags)
		return set(self.tagset.tag_to_string(tag) for tag in tags)
	
	def lemstrings_of_token(self, tok):
		"""Returns a set of unicode strings, corresponding to disamb lemmas
		found in the token."""
		if self.first_lex_only:
			# forcing first disamb lexeme only
			for lex in tok.lexemes():
				if lex.is_disamb():
					return set([unicode(lex.lemma())])
			assert False, 'no disamb lemma here'
		
		lemmas = set(
			unicode(lex.lemma())
			for lex in tok.lexemes()
			if lex.is_disamb())
		return lemmas
	
	def lem_concat(self, tok_seq, case_sens):
		"""Returns a concatenation of possibly lowercased disamb lemmas of
		tokens belonging to the given sequence. If any token has multiple
		different (lowercased) disamb lemmas, will surround them with
		parentheses and separate with |."""
		buff = StringIO()
		for tok in tok_seq:
			their_lemmas = self.lemstrings_of_token(tok)
			if case_sens:
				their_lc_lemmas = sorted(their_lemmas)
			else:
				their_lc_lemmas = sorted(
					set(lem.lower() for lem in their_lemmas))
			if len(their_lc_lemmas) > 1:
				# get u'(first|second|…)
				text = u'(%s)' % u'|'.join(their_lc_lemmas)
			else:
				text = their_lc_lemmas[0]
			buff.write(text)
		return buff.getvalue()
	
	def cmp_toks(self, tok1, tok2):
		"""Returns a set of features concerning comparison of two tokens:
		tok1 from tagger output and tok2 from ref. Actually, both should be
		interchangable, hence are labelled tok1 and tok2.
		This function is called only when token-to-token alignment is possible.
		"""
		hit_feats = set()
		tok1_lems = self.lemstrings_of_token(tok1)
		tok2_lems = self.lemstrings_of_token(tok2)
		tok1_lems_nocase = set(lem.lower() for lem in tok1_lems)
		tok2_lems_nocase = set(lem.lower() for lem in tok2_lems)
		tok1_tags = self.tagstrings_of_token(tok1)
		tok2_tags = self.tagstrings_of_token(tok2)
		tok1_pos = set(t.split(':', 1)[0] for t in tok1_tags)
		tok2_pos = set(t.split(':', 1)[0] for t in tok2_tags)
		# check lemmas in case-insensitive manner
		if tok1_lems_nocase == tok2_lems_nocase:
			hit_feats.add(Feat.STRONG_LEM_NOCASE_HIT)
		# check lemmas in case-sensitive manner
		if tok1_lems == tok2_lems:
			hit_feats.add(Feat.STRONG_LEM_HIT)
			hit_feats.add(Feat.WEAK_LEM_HIT)
		elif tok1_lems.intersection(tok2_lems):
			hit_feats.add(Feat.WEAK_LEM_HIT)
		# check POSes
		if tok1_pos == tok2_pos:
			hit_feats.add(Feat.STRONG_POS_HIT)
			hit_feats.add(Feat.WEAK_POS_HIT)
		elif tok1_pos.intersection(tok2_pos):
			hit_feats.add(Feat.WEAK_POS_HIT)
		# check full tags
		if tok1_tags == tok2_tags:
			hit_feats.add(Feat.STRONG_TAG_HIT)
			hit_feats.add(Feat.WEAK_TAG_HIT)
		elif tok1_tags.intersection(tok2_tags):
			hit_feats.add(Feat.WEAK_TAG_HIT)
		
		return hit_feats
	
	def update(self, tag_seq, ref_seq):
		self.tag_toks += len(tag_seq)
		#self.ref_toks += len(ref_seq) TODO
		
		# initialise empty feat set for each ref token
		pre_feat_sets = [set() for _ in ref_seq]
		# check if there are any "unknown words"
		for tok, feats in zip(ref_seq, pre_feat_sets):
			feats.add(Feat.UNKNOWN if self.is_unk(tok) else Feat.KNOWN)
		
		# now check for segmentation changes
		# first variant: no segmentation mess
		if len(tag_seq) == 1 and len(ref_seq) == 1:
			# there is only one token, hence one feat set
			# update it with hit feats
			pre_feat_sets[0].add(Feat.SEG_NOCHANGE)
			pre_feat_sets[0].update(self.cmp_toks(tag_seq[0], ref_seq[0]))
		else:
			if self.debug:
				print 'SEGCHANGE\t%s\t%s' % (
					text(tag_seq, True, True), text(ref_seq, True, True))
			
			# mark all as subjected to segmentation changes
			for feats in pre_feat_sets:
				feats.add(Feat.SEG_CHANGE)
			
			# now check lower-cased concatenation of ref and tag lemmas
			# if they both make up the same text (case-insensitively)
			# then we will count is is STRONG_LEM_NOCASE_HIT
			# this will be used only by the heuristic measure
			# that counts SEG_CHANGE tokens
			# NOTE: in case of lemma ambiguity there is not a simple way
			# to compare whole sequence; any lemma ambiguity will cause
			# the sequence to be unmatched
			if self.lem_concat(tag_seq, False) == self.lem_concat(ref_seq, False):
				for feats in pre_feat_sets:
					feats.add(Feat.STRONG_LEM_NOCASE_HIT)
			
			# the same with case-sensitive concatenation
			if self.lem_concat(tag_seq, True) == self.lem_concat(ref_seq, True):
				for feats in pre_feat_sets:
					feats.add(Feat.STRONG_LEM_HIT)
			#else:
				#print 'UUUU lem concat failed'
				#print 'TAG', self.lem_concat(tag_seq, True)
				#print 'REF', self.lem_concat(ref_seq, True)
			
			# check if all ref and tagged toks are punctuation
			#all_punc_ref = all(self.is_punc(tok) for tok in ref_seq)
			#all_punc_tag = all(self.is_punc(tok) for tok in tag_seq)
			#if all_punc_ref and all_punc_tag:
				#for feats in pre_feat_sets:
					#feats.update([Feat.ALLPUNC_HIT,
						#Feat.WEAK_POS_HIT, Feat.STRONG_POS_HIT,
						#Feat.WEAK_TAG_HIT, Feat.STRONG_TAG_HIT])
				## second variant: PUNC v. PUNC gives hit
				#if self.debug: print '\t\tpunc hit, ref len', len(ref_seq)
			#else:
				#nonpunc_ref = [tok for tok in ref_seq if not self.is_punc(tok)]
				#nonpunc_tag = [tok for tok in tag_seq if not self.is_punc(tok)]
				#if len(nonpunc_ref) == len(nonpunc_tag) == 1:
					## perhaps third variant: both seqs have one non-punc token
					## if the non-punc tokens match, will take the hit features
					## for the whole ref
					#hit_feats = self.cmp_toks(nonpunc_tag[0], nonpunc_ref[0])
					#for feats in pre_feat_sets:
						#feats.update(hit_feats)
					#if Feat.WEAK_TAG_HIT in hit_feats:
						#for feats in pre_feat_sets:
							#feats.add(Feat.PUNCAROUND_HIT)
						#if self.debug: print '\t\tpuncPLUS weak hit, ref len', len(ref_seq)
		# collect features assigned to each ref token in this aligned fragment
		for feats in pre_feat_sets:
			self.eat_ref_toks(feats, 1)
			if self.debug:
				print ' - ', ', '.join(sorted(feats))
	
	def count_all(self): # TODO remove
		"""Returns the number of all reference tokens."""
		return sum(self.tag_feat.values())
	
	def count_with(self, feats):
		"""Returns the number of reference tokens having all the given
		features."""
		satisfying = [key for key in self.tag_feat if key.issuperset(set(feats))]
		return sum(self.tag_feat[key] for key in satisfying)
	
	def percentage_with(self, feats, wrt_to = None):
		"""Returns the percentage of reference tokens that have the given
		features. By default all the reference tokens are treated as the
		denominator. If wrt_to given, will be used as a reference feat set."""
		count = self.count_with(feats)
		if wrt_to is not None:
			denom = self.count_with(wrt_to)
		else:
			denom = self.ref_toks
		if denom == 0:
			return 0.0 # what else can we do? should be labelled N/A
		return 100.0 * count / denom

	def value_of(self, metric):
		"""Calculates the value of the given metric, being a tuple of
		features for tokens counted as hit and features for tokens counted
		at all (or None if all the tokens should be counted). The first feats
		should be a subset of the second one."""
		return self.percentage_with(metric[0], metric[1])
	
	def dump(self):
		print '----'
		print 'REF-toks\t%d' % self.ref_toks
		for m_name in dir(Metric):
			if not m_name.startswith('_'):
				metric = getattr(Metric, m_name)
				print '%s\t%.4f%%' % (m_name, self.value_of(metric))
		# calc upper bound
		upbound = self.value_of(Metric.WC_LOWER) + self.value_of(Metric.SEG_CHANGE)
		print 'WC_UPPER\t%.4f%%' % upbound

def go():
	parser = OptionParser(usage=descr)
	parser.add_option('-i', '--input-format', type='string', action='store',
		dest='input_format', default='xces',
		help='set the input format; default: xces')
	parser.add_option('-t', '--tagset', type='string', action='store',
		dest='tagset', default='nkjp',
		help='set the tagset used in input; default: nkjp')
	parser.add_option('-q', '--quiet', action='store_false', default=True, dest='verbose')
	parser.add_option('-u', '--unk-tag', type='string', action='store',
		dest='unk_tag', default='ign',
		help='set the tag used for unknown forms; default: ign')
	parser.add_option('-k', '--keep-optional', action='store_false',
		default=True, dest='expand_optional',
		help='do not expand unspecified optional attributes to multiple tags')
	parser.add_option('-s', '--ignore-spaces', action='store_false',
		default=True, dest='respect_spaces',
		help='ignore spaces between tokens when comparing')
	parser.add_option('-f', '--first-lexeme-only', action='store_true',
		default=False, dest='first_lex_only',
		help='read only each token\'s first disamb lexeme (tag+lemma)')
	parser.add_option('-d', '--debug', action='store_true', dest='debug_mode')
	(options, args) = parser.parse_args()
	
	if len(args) < 2 or len(args) % 2 != 0:
		print 'You need to provide a series of tagged folds and a coresponding'
		print 'series of reference folds.'
		print 'See --help for details.'
		print
		sys.exit(1)
	
	tagset = corpus2.get_named_tagset(options.tagset)
	
	num_folds = len(args) / 2
	
	weak_lem_lower_bound = 0.0
	kn_strong_lem_lower_bound = 0.0
	unk_strong_lem_lower_bound = 0.0
	strong_lem_lower_bound = 0.0
	strong_lem_nocase_lower_bound = 0.0
	strong_lem_case_cat_heur = 0.0
	strong_lem_nocase_cat_heur = 0.0
	
	weak_lower_bound = 0.0
	weak_upper_bound = 0.0
	
	strong_pos_lower = 0.0

	unk_weak_lower = 0.0
	unk_weak_upper = 0.0
	kn_weak_lower = 0.0
	kn_weak_upper = 0.0
	
	perc_unk = 0.0
	perc_segchange = 0.0
	
	for fold_idx in range(num_folds):
		tag_fn = args[fold_idx] # filename of tagged fold @ fold_idx
		ref_fn = args[fold_idx + num_folds] # ... reference fold @ fold_idx
		if options.verbose:
			print '### FOLD %2d: %s (tag) v. %s (ref)' % ((fold_idx + 1), tag_fn, ref_fn)
		tag_rdr = corpus2.TokenReader.create_path_reader(options.input_format, tagset, tag_fn)
		ref_rdr = corpus2.TokenReader.create_path_reader(options.input_format, tagset, ref_fn)
		
		res = TokComp(
			tagset, options.unk_tag, options.expand_optional,
			options.first_lex_only, options.debug_mode)
		for tag_seq, ref_seq in tok_seqs(tag_rdr, ref_rdr, options.respect_spaces, options.verbose, options.debug_mode):
			res.update(tag_seq, ref_seq)

		print "PolEval 2017 competition scores"
		print "-------------------------------"
		print 'POS accuracy (Subtask A score): \t%.4f%%' % res.value_of(Metric.SC_LOWER)
		print 'POS accuracy (known words): \t%.4f%%' % res.value_of(Metric.KN_SC_LOWER)
		print 'POS accuracy (unknown words): \t%.4f%%' % res.value_of(Metric.UNK_SC_LOWER)
		print 'Lemmatization accuracy (Subtask B score): \t%.4f%%' % res.value_of(Metric.SL_LOWER)
		print 'Lemmatization accuracy (known words): \t%.4f%%' % res.value_of(Metric.KN_SL_LOWER)
		print 'Lemmatization accuracy (unknown words): \t%.4f%%' % res.value_of(Metric.UNK_SL_LOWER)
		print 'Overall accuracy (Subtask C score): \t%.4f%%' % ((res.value_of(Metric.SC_LOWER) + res.value_of(Metric.SL_LOWER)) / 2)


		if options.verbose:
			res.dump()
		
		weak_lem_lower_bound += res.value_of(Metric.WL_LOWER)
		kn_strong_lem_lower_bound += res.value_of(Metric.KN_SL_LOWER)
		unk_strong_lem_lower_bound += res.value_of(Metric.UNK_SL_LOWER)
		strong_lem_lower_bound += res.value_of(Metric.SL_LOWER)
		
		strong_lem_nocase_lower_bound += res.value_of(Metric.SL_NOCASE_LOWER)
		
		strong_lem_case_cat_heur += res.value_of(Metric.SL_CASE_CAT_HEUR)
		strong_lem_nocase_cat_heur += res.value_of(Metric.SL_NOCASE_CAT_HEUR)
		
		weak_lower_bound += res.value_of(Metric.WC_LOWER)
		weak_upper_bound += res.value_of(Metric.WC_LOWER) + res.value_of(Metric.SEG_CHANGE)
		unk_weak_lower += res.value_of(Metric.UNK_WC_LOWER)
		unk_weak_upper += res.value_of(Metric.UNK_WC_LOWER) + res.value_of(Metric.UNK_SEG_CHANGE)
		kn_weak_lower += res.value_of(Metric.KN_WC_LOWER)
		kn_weak_upper += res.value_of(Metric.KN_WC_LOWER) + res.value_of(Metric.KN_SEG_CHANGE)
		strong_pos_lower += res.value_of(Metric.POS_SC_LOWER)
		perc_unk += res.value_of(Metric.UNK)
		perc_segchange += res.value_of(Metric.SEG_CHANGE)
	
	# weak lemma -- when sets of possible lemmas output and in ref corp intersect
	print 'AVG weak lemma lower bound\t%.4f%%' % (weak_lem_lower_bound / num_folds)
	print 'AVG KN strong lemma lower bound\t%.4f%%' % (kn_strong_lem_lower_bound / num_folds)
	print 'AVG UNK strong lemma lower bound\t%.4f%%' % (unk_strong_lem_lower_bound / num_folds)
	# strong lemma -- when sets of possible lemmas output and in ref corp are equal
	print 'AVG strong lemma lower bound\t%.4f%%' % (strong_lem_lower_bound / num_folds)
	print 'AVG strong lemma nocase lower bound\t%.4f%%' % (strong_lem_nocase_lower_bound / num_folds)
	
	print 'AVG strong lemma case concat heur\t%.4f%%' % (strong_lem_case_cat_heur / num_folds)
	print 'AVG strong lemma nocase concat heur\t%.4f%%' % (strong_lem_nocase_cat_heur / num_folds)
	
	print 'AVG weak corr lower bound\t%.4f%%' % (weak_lower_bound / num_folds)
	print 'AVG weak corr upper bound\t%.4f%%' % (weak_upper_bound / num_folds)
	
	print 'AVG UNK weak corr lower bound\t%.4f%%' % (unk_weak_lower / num_folds)
	print 'AVG UNK weak corr upper bound\t%.4f%%' % (unk_weak_upper / num_folds)
	print 'AVG KN  weak corr lower bound\t%.4f%%' % (kn_weak_lower / num_folds)
	print 'AVG KN  weak corr upper bound\t%.4f%%' % (kn_weak_upper / num_folds)
	print 'AVG POS strong corr lower bound\t%.4f%%' % (strong_pos_lower / num_folds)
	
	print 'AVG percentage UNK\t%.4f%%' % (perc_unk / num_folds)
	print 'AVG percentage seg change\t%.4f%%' % (perc_segchange / num_folds)
	

if __name__ == '__main__':
	go()
