Issue 209 ALE "CDS LG-E-clean" 2019-04-26 400 clusters MWC=1

Issue 209 GL exception 'list index out of range'

"Child Directed Speech" corpus, "LG-E-clean" dataset,
trash filter off: min_word_count = 1, max_sentence_length off, Link Grammar 5.5.1.
Server 94, fresh clone of singnet repository 2019-04-03, fresh ull environment ~ 2018-03-18

Basic settings

In [1]:
import os, sys, time
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path: sys.path.append(module_path)
from src.grammar_learner.utl import UTC, test_stats
from src.grammar_learner.read_files import check_dir, check_corpus
from src.grammar_learner.write_files import list2file
from src.grammar_learner.widgets import html_table
from src.grammar_learner.pqa_table import table_rows, params, wide_rows
tmpath = module_path + '/tmp/'
check_dir(tmpath, True, 'none')
start = time.time()
runs = (1,1)
print(UTC(), ':: module_path:', module_path)

2019-04-26 07:05:07 UTC :: module_path: /home/obaskov/94/ULL

Corpus test settings

In [2]:
corpus = 'CDS' # 'Child Directed Speech' 
dataset = 'LG-E-clean'
kwargs = {
    'max_sentence_length':  99      ,
    'max_unparsed_words' :  10      ,
    'left_wall'     :   ''          ,
    'period'        :   False       ,
    'context'       :   1           ,   # 1: connector-based word space
    'min_word_count':   1           ,   # 31,21,11,6,1
    'word_space'    :   'sparse'    ,
    'clustering'    :   ['agglomerative', 'ward'],
    'clustering_metric' : ['silhouette', 'cosine'],
    'cluster_range' :   [400]       ,
    'top_level'     :   0.01        ,
    'grammar_rules' :   2           ,   # disjunct-based grammar rules
    'max_disjuncts' :   1000000     ,   # off
    'stop_words'    :   []          ,
    'tmpath'        :   tmpath      ,
    'verbose'       :   '+'      ,
    'template_path' :   'poc-turtle',
    'linkage_limit' :   1000        }
rp = module_path + '/data/CDS/LG-E-clean'
cp = rp  # corpus path = reference_path
out_dir = module_path + '/output/' + '_issue_209_CDS-ALE-ILE-' + str(UTC())[:10] +'_'
print(UTC(), '\n', out_dir)

2019-04-26 07:05:07 UTC 
 /home/obaskov/94/ULL/output/_issue_209_CDS-ALE-ILE-2019-04-26_

Check IndexError in case requested N clusters > N unique words

In [3]:
line = [['', corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
header[0] = ''
display(html_table([header] + a)); print(test_stats(log))

Overal execution:   0%|          | 0/3037 [00:00<?, ?it/s]
CDS-LG-English-clean.ull:   0%|          | 0/3037 [00:00<?, ?sentences/s]

Overal execution: 100%|##########| 3037/3037 [00:01<00:00, 2655.24it/s]
CDS-LG-English-clean.ull: 100%|##########| 3037/3037 [00:01<00:00, 2772.44sentences/s]


Overal execution: 100%|##########| 3037/3037 [00:01<00:00, 2655.24it/s]
CorpusParsingSpaceLinkageAffinityG12nThresholdRulesMWCNNSIPAPQF1Top 5 cluster sizes
CDSLG-E-cleancALWEdwardeuclideannone---3011---0.0100%99%0.99[1, 0]
Cleaned dictionary: 301 words, grammar learn time: 00:00:05, grammar test time: 00:00:01

Raise ValueError in case of wrong corpus filtering parameters

In [4]:
kwargs['max_sentence_length'] = 1
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
display(html_table([header] + a)); print(test_stats(log))

--- Logging error ---
Traceback (most recent call last):
  File "/home/obaskov/94/ULL/src/grammar_learner/pqa_table.py", line 397, in wide_rows
    rulez, re = learn(**kwargs)
  File "/home/obaskov/94/ULL/src/grammar_learner/learner.py", line 98, in learn
    + str(kwa(0, 'max_unparsed_words', **kwargs)))
ValueError: Empty filtered dataset with max_sentence_length = 1, max_unparsed_words = 10

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/logging/__init__.py", line 993, in emit
    msg = self.format(record)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/logging/__init__.py", line 839, in format
    return fmt.format(record)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/logging/__init__.py", line 576, in format
    record.message = record.getMessage()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/logging/__init__.py", line 338, in getMessage
    msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 486, in start
    self.io_loop.start()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 132, in start
    self.asyncio_loop.run_forever()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/asyncio/base_events.py", line 422, in run_forever
    self._run_once()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/asyncio/base_events.py", line 1434, in _run_once
    handle._run()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/asyncio/events.py", line 145, in _run
    self._callback(*self._args)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/tornado/ioloop.py", line 758, in _run_callback
    ret = callback()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/tornado/stack_context.py", line 300, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 536, in <lambda>
    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
    self._handle_recv()
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
    self._run_callback(callback, msg)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
    callback(*args, **kwargs)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/tornado/stack_context.py", line 300, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
    raw_cell, store_history, silent, shell_futures)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2901, in run_ast_nodes
    if self.run_code(code, result):
  File "/home/obaskov/miniconda3/envs/ull/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2961, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-4-d373431b40d8>", line 2, in <module>
    a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
  File "/home/obaskov/94/ULL/src/grammar_learner/pqa_table.py", line 419, in wide_rows
    sys.exc_info())
Message: 'pqa_table.py wide_rows:'
Arguments: ('learn_grammar(**kwargs) ⇒ exception:\n', (<class 'ValueError'>, ValueError('Empty filtered dataset with max_sentence_length = 1, max_unparsed_words = 10',), <traceback object at 0x7fda415e3948>))

---------------------------------------------------------------------------
UnboundLocalError                         Traceback (most recent call last)
<ipython-input-4-d373431b40d8> in <module>()
      1 kwargs['max_sentence_length'] = 1
----> 2 a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
      3 display(html_table([header] + a)); print(test_stats(log))

~/94/ULL/src/grammar_learner/pqa_table.py in wide_rows(lines, out_dir, cp, rp, runs, **kwargs)
    480         average.append(avg_line)
    481 
--> 482     re.update({'grammar_test_time': sec2string(time.time() - start)})
    483 
    484     stats = []

UnboundLocalError: local variable 're' referenced before assignment