Compatibility fixes + setup changes (#58)

* Appveyor py310 and skip middling

* Try this

* Please

* Mucking around with tox and versions

* blech

* Now?

* Why do you hate me?

* Is it you?

* Hmmm

* Mebbeh?

* Weirdness

* Do it mah dude

* Blerg?

* More blerg

* Please

* Hmm?

* Getting minimum pytorch version working.

* Updated CHANGELOG + formatting

Author: sdrobert

.appveyor.yml

@@ -0,0 +1,40 @@
+version: build.{build}.branch.{branch}
+# FIXME(sdrobert): tox-ltt doesn't work on Windows as of yet, causing a
+# virtualenv error. Until it's fixed we'll use the Ubuntu image.
+image: Ubuntu
+
+environment:
+  matrix:
+    - TOXENV: py36-earliest
+      PYTHON: "3.6"
+    - TOXENV: py37
+      PYTHON: "3.7"
+    - TOXENV: py38
+      PYTHON: "3.8"
+    - TOXENV: py39
+      PYTHON: "3.9"
+    # - TOXENV: py310  # wheel not available yet
+    #   PYTHON: "3.10"
+
+stack: python %PYTHON%
+
+branches:
+  except:
+    - /docs/
+
+for:
+  -
+    matrix:
+      only:
+        - PYTHON: "3.7"
+        - PYTHON: "3.8"
+    skip_non_tags: true
+
+build: off
+
+install:
+  - python3 -m pip install -U pip virtualenv setuptools wheel six
+  - python3 -m pip install -U tox tox-ltt
+
+test_script:
+  - python3 -m tox

.gitignore

@@ -106,4 +106,4 @@ venv.bak/
 .ftpignore
 .ftpconfig
 .vscode
-version.py
+_version.py

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## HEAD
 
+- Removed `setup.py`.
+- Deleted conda recipe in prep for [conda-forge](https://conda-forge.org/).
+- Compatibility/determinism fixes for 1.5.1.
+- Bump minimum PyTorch version to 1.5.1. Actually testing this minimum!
+- `version.py` -> `_version.py`.
 - A number of modifications and additions related to decoding and language
   models, including:
   - `beam_search_advance` has been simplified, with much of the end-of-sequence

appveyor.yml

@@ -1,3 +1,6 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]
 build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+write_to = "src/pydrobert/torch/_version.py"

pyproject.toml

@@ -23,7 +23,7 @@ package_dir =
 python_requires = >= 3.6
 install_requires =
   numpy
-  torch>=1.0.1
+  torch>=1.5.1
   param
 
 [options.entry_points]

recipe/meta.yaml

@@ -18,7 +18,7 @@
 __copyright__ = "Copyright 2021 Sean Robertson"
 
 try:
-    from .version import version as __version__  # type: ignore
+    from ._version import version as __version__  # type: ignore
 except ImportError:
     __version__ = "inplace"
 

setup.cfg

@@ -28,11 +28,6 @@
 
 import torch
 
-try:
-    torch_bool = torch.bool
-except AttributeError:
-    torch_bool = torch.uint8
-
 __all__ = [
     "to_z",
     "to_b",
@@ -425,11 +420,10 @@ def _to_z_tilde(logits, b, dist):
             -torch.log(-log_v / theta - log_v.gather(-1, b[..., None])),
         )
     elif dist in ONEHOT_SYNONYMS:
-        b = b.byte()
         theta = torch.softmax(logits, dim=-1)
         log_v = v.log()
         z_tilde = torch.where(
-            b,
+            b.bool(),
             -torch.log(-log_v),
             -torch.log(-log_v / theta - log_v.gather(-1, b.argmax(-1, keepdim=True))),
         )

setup.py

@@ -1194,7 +1194,7 @@ def forward(
                 f"Expected dim 2 of logits to be {self.lm.vocab_size + 1}, got {Vp1}"
             )
         if lens is None:
-            lens = torch.full((N,), T, device=logits.device)
+            lens = torch.full((N,), T, device=logits.device, dtype=torch.long)
             len_min = len_max = T
         elif lens.dim() != 1:
             raise RuntimeError("lens must be 1 dimensional")
@@ -1211,7 +1211,9 @@ def forward(
         y_prev_lens = y_prev_last = torch.zeros(
             (N, 1), dtype=torch.long, device=logits.device
         )
-        prev_is_prefix = torch.full((N, 1, 1), True, device=logits.device)
+        prev_is_prefix = torch.full(
+            (N, 1, 1), True, device=logits.device, dtype=torch.bool
+        )
         if self.lm is not None:
             prev = self.lm.update_input(prev, y_prev)
         prev_width = 1

src/pydrobert/torch/__init__.py

@@ -249,7 +249,7 @@ def beam_search_advance(
     K = min(width, Kp * V)
     cand_log_probs = (log_probs_prev.unsqueeze(2) + log_probs_t).flatten(1)
     log_probs_next, next_ind = cand_log_probs.topk(K, 1)
-    next_src = torch.div(next_ind, V, rounding_mode="trunc")
+    next_src = next_ind.floor_divide(V)
     next_token = (next_ind % V).unsqueeze(0)  # (1, N, K)
 
     if tm1:
@@ -578,7 +578,9 @@ def ctc_prefix_search_advance(
     del tot_probs_cand
 
     next_is_nonext = next_ind >= (Kp * V)
-    next_src = torch.where(next_is_nonext, next_ind - (Kp * V), next_ind // V)
+    next_src = torch.where(
+        next_is_nonext, next_ind - (Kp * V), next_ind.floor_divide(V)
+    )
     next_ext = next_ind % V
 
     y_next_prefix_lens = y_prev_lens.gather(1, next_src)  # (N, K)
@@ -1967,7 +1969,7 @@ def pad_variable(
     arange_ = torch.arange(Tp, device=x.device)
     left_mask = (pad[0].unsqueeze(1) > arange_).unsqueeze(2).expand(N, Tp, F)
     if mode == "constant":
-        buff = torch.tensor(value, device=x.device, dtype=x.dtype).view(1)
+        buff = torch.tensor(value, device=x.device).to(x.dtype).view(1)
         left_pad = buff.expand(pad[0].sum() * F)
         right_pad = buff.expand(pad[1].sum() * F)
     elif mode == "reflect":
@@ -2172,8 +2174,11 @@ def _string_matching(
     assert not exclude_last or (return_mask or return_prf_dsts)
     if ref.dim() != 2 or hyp.dim() != 2:
         raise RuntimeError("ref and hyp must be 2 dimensional")
+    mult = 1.0
     if ins_cost == del_cost == sub_cost > 0.0:
         # results are equivalent and faster to return
+        if not return_mistakes:
+            mult = ins_cost
         ins_cost = del_cost = sub_cost = 1.0
         return_mistakes = False
     elif return_mistakes and warn:
@@ -2220,8 +2225,12 @@ def _string_matching(
                 hyp_lens = hyp_lens - hyp_eq_mask.to(hyp_lens.dtype)
             del ref_eq_mask, hyp_eq_mask
     else:
-        ref_lens = torch.full((batch_size,), max_ref_steps, device=ref.device)
-        hyp_lens = torch.full((batch_size,), max_hyp_steps, device=ref.device)
+        ref_lens = torch.full(
+            (batch_size,), max_ref_steps, device=ref.device, dtype=torch.long
+        )
+        hyp_lens = torch.full(
+            (batch_size,), max_hyp_steps, device=ref.device, dtype=torch.long
+        )
     ins_cost = torch.tensor(float(ins_cost), device=device)
     del_cost = torch.tensor(float(del_cost), device=device)
     sub_cost = torch.tensor(float(sub_cost), device=device)
@@ -2345,6 +2354,7 @@ def _string_matching(
         )
         return mask
     elif return_prf_dsts:
+        prefix_ers = prefix_ers * mult
         if norm:
             prefix_ers = prefix_ers / ref_lens.to(row.dtype)
             zero_mask = ref_lens.eq(0).unsqueeze(0)
@@ -2381,6 +2391,7 @@ def _string_matching(
         er = mistakes.gather(0, ref_lens.unsqueeze(0)).squeeze(0)
     else:
         er = row.gather(0, ref_lens.unsqueeze(0)).squeeze(0)
+    er = er * mult
     if norm:
         er = er / ref_lens.to(er.dtype)
         zero_mask = ref_lens.eq(0)

src/pydrobert/torch/estimators.py

@@ -767,9 +767,7 @@ def update_input(self, prev, hist):
         def calc_idx_log_probs(self, hist, prev, idx):
             idx_zero = idx == 0
             if idx_zero.all():
-                x = torch.arange(hist.size(0), device=hist.device).clamp(
-                    max=self.vocab_size
-                )
+                x = torch.arange(hist.size(0), device=hist.device)
             elif not idx.dim():
                 x = hist[idx - 1]
             else:
@@ -785,8 +783,8 @@ def calc_idx_log_probs(self, hist, prev, idx):
                 {"hidden_state": h_1, "cell_state": c_1},
             )
 
-    T, N, V, K = 64, 16, 32, 8
-    assert K <= V and N <= V
+    T, N, V, K = 64, 16, 128, 8
+    assert K <= V and N * K <= V
     lm = RNNLM(V)
     search = layers.BeamSearch(lm, K, eos=0, max_iters=T).to(device)
     y_prev = torch.arange(N, device=device)