Create pkl (words) for context up to n=8; add bz2 compression

Author: Peter Johnson

evaluation_function/models/shannon_words_ngram.py

@@ -28,36 +28,36 @@ def log(msg):
 # If creating when deployed: 
 #FILE = Path(tempfile.gettempdir()) / "ngram_counts.pkl"
 # If creating locally, to be copied when deployed:
-FILE = MODEL_DIR / "ngram_counts.pkl"
+FILE_BASE = MODEL_DIR / "ngram_counts.pkl.bz2" 
 
 def get_counts(n=3, dev=False):
     print(f"Loading/building n-gram counts for n={n}...")
-    if os.path.exists(FILE):
+    FILE_NAME = FILE_BASE.with_name(FILE_BASE.stem + f"_{n:02d}" + "".join(FILE_BASE.suffixes))
+    if os.path.exists(FILE_NAME):
         try:
-            with open(FILE, "rb") as f:
+            with bz2.BZ2File(FILE_NAME, "rb") as f:
                 cache = pickle.load(f)
             if not isinstance(cache, dict):
                 raise RuntimeError(f"Loaded cache is {type(cache)}, not dict — contents: {str(cache)[:300]}")
-            if n not in cache:
-                raise RuntimeError(f"Loaded keys={list(cache.keys())[:10]} (len={len(cache)}) — expected {n}")
+            #if n not in cache:
+            #    raise RuntimeError(f"Loaded keys={list(cache.keys())[:10]} (len={len(cache)}) — expected {n}")
         except Exception as e:
-            raise RuntimeError(f"Failed to load {FILE}: {e}")        
+            raise RuntimeError(f"Failed to load {FILE_NAME}: {e}")        
     elif dev: # from here the deployed version will not work because the corpora are not bundled (to save space)
         cache = {}
         print(f"Building n-gram counts from NLTK corpora (dev mode)")
         try:
             if n not in cache:
-                print(f"Building n={n} counts...")
-                cache[n] = build_counts(n, START, END)  # only works if NLTK corpora are available           
-                print(f"Saving n-gram counts to {FILE}...") 
-                with open(FILE, "wb") as f:
-                    pickle.dump(cache,f)
+                print(f"Starting building counts up to n={n} ...")
+                build_counts(FILE_BASE, list(range(1, n + 1)), START, END)  # only works if NLTK corpora are available           
+                with bz2.BZ2File(FILE_NAME, "rb") as f:
+                    cache = pickle.load(f)
         except Exception as e:
             raise RuntimeError(f"Failed to rebuild or save n-gram counts {e}")
     else:
         raise FileNotFoundError(f"N-gram counts file not found at {FILE}, and dev mode is off so counts not generated.")    
     print(f"Loaded cache is {type(cache)}, — contents: {str(cache)[:300]}")
-    counts = cache[n]
+    counts = cache
     if n == 1:
         counts.setdefault((), {})                 # CHANGE: ensure unigram context exists
         counts[()].pop(END, None)                 # CHANGE: drop </s> if present (old caches)

evaluation_function/models/storage/ngram_counts.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5d8b1c820f7c17ec499651e125e48f3e02c0cc8bf524edf69259541ca7bbe4e
+size 451494

evaluation_function/models/storage/ngram_counts.pkl_01.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be05894ea4878520850161b7ba864b1f68f1780f571232f0d7c3e618f3232462
+size 3711212

evaluation_function/models/storage/ngram_counts.pkl_02.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc26a3940cf9ab4ab85f48900218320997a7f184cd8e2e345dce39e8e2f64c17
+size 14369999

evaluation_function/models/storage/ngram_counts.pkl_03.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:600106349b271d69ee3dfc5d9168f03e78239788d7bb30ea0a4e2c0c3d7db0b1
+size 31263714

evaluation_function/models/storage/ngram_counts.pkl_04.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5e5e2f72d6740d8572ae5a8d41d6a39f511f7e809c9249a42678b78cf990081
+size 39222247

evaluation_function/models/storage/ngram_counts.pkl_05.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc16262131b93f909597bf950815d7f85703645182b5a2fd8be30c1b3b0f74db
+size 41750737

evaluation_function/models/storage/ngram_counts.pkl_06.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741e9fda02f42409da78d59c2959b3d6c273a451953d5ebeb8a870736f9e8d84
+size 43868262

evaluation_function/models/storage/ngram_counts.pkl_07.pkl.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c865ca124783d95c67ed7bbd593d805fc1f5f235ee02ef7285adb8781b6d6dbb
+size 44082806