add basic test of custom SNPP variant #8

Author: virgesmith

README.md

@@ -6,7 +6,7 @@
 
 # ukpopulation: UK Demographic Projections
 
-> ## Latest news: upcoming 1.1 release
+> ## Latest news: 1.1 release
 > - adds UK household projections
 > - initial support for custom SNPP variants
 > - better consistency across the MYE/NPP/SNPP APIs (breaks backwards compatibility)

requirements.txt

@@ -5,4 +5,6 @@ openpyxl
 beautifulsoup4
 lxml
 pyexcel
+pyexcel-xls
+pyexcel-xlsx
 ukcensusapi

setup.py

@@ -23,6 +23,8 @@ def readme():
                     "beautifulsoup4",
                     "lxml",
                     "pyexcel",
+                    "pyexcel-xls",
+                    "pyexcel-xlsx",
                     "ukcensusapi"],
   classifiers=(
     "Programming Language :: Python :: 3",

tests/raw_data/test_scenario.csv

@@ -0,0 +1,217 @@
+GEOGRAPHY_CODE,PEOPLE,PEOPLE_ppp,PROJECTED_YEAR_NAME,net_delta
+E06000001,92845.0,92845.0,2016,0.0
+E06000005,106827.0,106327.0,2016,500.0
+E06000047,521776.0,521776.0,2016,0.0
+S12000033,229840.0,229840.0,2016,0.0
+S12000034,262190.0,262190.0,2016,0.0
+S12000041,116520.0,116520.0,2016,0.0
+W06000011,244462.0,244462.0,2016,0.0
+W06000016,238179.0,238179.0,2016,0.0
+W06000018,180453.0,180453.0,2016,0.0
+E06000001,92945.0,92945.0,2017,0.0
+E06000005,106833.0,106333.0,2017,500.0
+E06000047,523458.0,523458.0,2017,0.0
+S12000033,234284.0,234284.0,2017,0.0
+S12000034,266756.0,266756.0,2017,0.0
+S12000041,117043.0,117043.0,2017,0.0
+W06000011,243950.84403700012,243950.84403700012,2017,0.0
+W06000016,238041.52757199996,238041.52757199996,2017,0.0
+W06000018,180754.05527699992,180754.05527699992,2017,0.0
+E06000001,93071.0,93071.0,2018,0.0
+E06000005,106876.0,106376.0,2018,500.0
+E06000047,525213.0,525213.0,2018,0.0
+S12000033,235986.0,235986.0,2018,0.0
+S12000034,268988.0,268988.0,2018,0.0
+S12000041,117207.0,117207.0,2018,0.0
+W06000011,244867.48761899996,244867.48761899996,2018,0.0
+W06000016,238481.94462,238481.94462,2018,0.0
+W06000018,181026.5377660001,181026.5377660001,2018,0.0
+E06000001,93168.0,93168.0,2019,0.0
+E06000005,106936.0,106436.0,2019,500.0
+E06000047,526898.0,526898.0,2019,0.0
+S12000033,237568.0,237568.0,2019,0.0
+S12000034,271340.0,271340.0,2019,0.0
+S12000041,117427.0,117427.0,2019,0.0
+W06000011,245801.46098000003,245801.46098000003,2019,0.0
+W06000016,238944.782903,238944.782903,2019,0.0
+W06000018,181296.4781450001,181296.4781450001,2019,0.0
+E06000001,93256.0,93256.0,2020,-1.9577669338018808
+E06000005,107028.0,106528.0,2020,500.0
+E06000047,528504.0,528504.0,2020,-4.732488171064978
+S12000033,239199.0,239199.0,2020,-2.043353762119705
+S12000034,273703.0,273703.0,2020,-1.4581288099724694
+S12000041,117628.0,117628.0,2020,-1.2076363359483897
+W06000011,246751.63779,246751.63779000004,2020,-6.112628892259426
+W06000016,239431.161604,239431.161604,2020,-5.833028890049543
+W06000018,181558.12861199985,181558.12861199985,2020,-5.060873586709353
+E06000001,93328.0406795495,93330.0,2021,-3.837290484042173
+E06000005,107077.0,106577.0,2021,500.0
+E06000047,529944.2545725788,529949.0,2021,-9.279340735101258
+S12000033,240865.94238883114,240868.0,2021,-3.9984621201745214
+S12000034,276118.52899483376,276120.0,2021,-2.8519736649729084
+S12000041,117842.79014608449,117844.0,2021,-2.366194278694432
+W06000011,247713.68015999606,247719.81677299997,2021,-11.957023998154561
+W06000016,239929.6658831305,239935.511199,2021,-11.4199285825653
+W06000018,181807.02989510386,181812.09784800012,2021,-9.912964963130397
+E06000001,93373.1606964968,93377.0,2022,-5.639814641510767
+E06000005,107102.0,106602.0,2022,500.0
+E06000047,531368.695554337,531378.0,2022,-13.653038135549185
+S12000033,242475.97474393732,242480.0,2022,-5.868377732811925
+S12000034,278559.1227882064,278562.0,2022,-4.182313128125209
+S12000041,118070.6291832812,118073.0,2022,-3.4759809347074144
+W06000011,248697.84670926,248709.8518180002,2022,-17.53796136702209
+W06000016,240443.54932637108,240454.9942590002,2022,-16.763576921037362
+W06000018,182046.64961961203,182056.576191,2022,-14.55683542723418
+E06000001,93397.35838304037,93403.0,2023,-7.378365522589459
+E06000005,107096.0,106596.0,2023,500.0
+E06000047,532841.308772616,532855.0,2023,-17.87485666830248
+S12000033,244119.09171273344,244125.0,2023,-7.669198893476124
+S12000034,280970.78141466196,280975.0,2023,-5.459552360578837
+S12000041,118338.51602995073,118342.0,2023,-4.544287463009017
+W06000011,249687.9169499454,249705.525972,2023,-22.89327042600369
+W06000016,240960.14592094198,240976.94668600013,2023,-21.90042670675642
+W06000018,182272.203979238,182286.78001600003,2023,-19.02476113093359
+E06000001,93420.61921379813,93428.0,2024,-9.057718742686463
+E06000005,107093.0,106593.0,2024,500.0
+E06000047,534320.0749348724,534338.0,2024,-21.958623910263626
+S12000033,245722.28019311855,245730.0,2024,-9.406020249461235
+S12000034,283381.4934980177,283387.0,2024,-6.688239291723472
+S12000041,118613.44498014366,118618.0,2024,-5.575151202199038
+W06000011,250673.4940499297,250696.479793,2024,-28.045332767275553
+W06000016,241464.52904899628,241486.47731300013,2024,-26.84959454033553
+W06000018,182470.32366293282,182489.371091,2024,-23.334875312310523
+E06000001,93414.94195346878,93424.0,2025,-9.002557134745063
+E06000005,107052.0,106552.0,2025,500.0
+E06000047,535826.9785427956,535849.0,2025,-21.841940361970003
+S12000033,247346.53144247644,247356.0,2025,-9.340778377484495
+S12000034,285749.2557187011,285756.0,2025,-6.635442204416523
+S12000041,118887.41170895935,118893.0,2025,-5.539375977458611
+W06000011,251648.98746982522,251677.145091,2025,-27.82919049489216
+W06000016,241950.5225686421,241977.42919499998,2025,-26.66334096084311
+W06000018,182637.94658828396,182661.305887,2025,-23.185226899578147
+E06000001,93405.99743727119,93415.0,2026,-8.947881230469742
+E06000005,107011.0,106511.0,2026,500.0
+E06000047,537318.0963842731,537340.0,2026,-21.727351657534598
+S12000033,248980.59715778247,248990.0,2026,-9.277939913220509
+S12000034,288074.3104118823,288081.0,2026,-6.585932643995398
+S12000041,119146.44829593832,119152.0,2026,-5.505218674954501
+W06000011,252610.8656812975,252638.804333,2026,-27.62295648884706
+W06000016,242414.71050246875,242441.4279419999,2026,-26.48770294511191
+W06000018,182776.11168066046,182799.3173930001,2026,-23.044951218397927
+E06000001,93403.05154375057,93412.0,2027,-8.894888622154554
+E06000005,106970.0,106470.0,2027,500.0
+E06000047,538766.2132103429,538788.0,2027,-21.613333829862203
+S12000033,250627.6603363773,250637.0,2027,-9.215274964907724
+S12000034,290319.3625894025,290326.0,2027,-6.537185408996379
+S12000041,119377.48385134034,119383.0,2027,-5.472140781415146
+W06000011,253545.8860431111,253573.6142759999,2027,-27.425792644858713
+W06000016,242851.12040752484,242877.658695,2027,-26.31902936080955
+W06000018,182881.54184721733,182904.60299900008,2027,-22.91210999083917
+E06000001,93387.10578290511,93396.0,2028,-8.842586808713902
+E06000005,106939.0,106439.0,2028,500.0
+E06000047,540098.3323571858,540120.0,2028,-21.501707490781268
+S12000033,252243.7249632678,252253.0,2028,-9.154036239512797
+S12000034,292493.4137126953,292500.0,2028,-6.490304739678956
+S12000041,119604.51720091805,119610.0,2028,-5.438921664047919
+W06000011,254457.32651418945,254484.85387400005,2028,-27.234141245068308
+W06000016,243262.68644363104,243289.05293400012,2028,-26.155411316373797
+W06000018,182953.8294425257,182976.75348099988,2028,-22.78264162279014
+E06000001,93364.15874883188,93373.0,2029,-8.791022487568018
+E06000005,106889.0,106389.0,2029,500.0
+E06000047,541316.4489404387,541338.0,2029,-21.396253295071205
+S12000033,253847.7874172965,253857.0,2029,-9.092903365992765
+S12000034,294581.4632172793,294588.0,2029,-6.445430043643098
+S12000041,119816.55118847615,119822.0,2029,-5.407206038465481
+W06000011,255347.16607938663,255374.498384,2029,-27.047062271110267
+W06000016,243648.2615739816,243674.4612589999,2029,-25.9965782678766
+W06000018,182992.34817676683,183015.13845300005,2029,-22.657591126582567
+E06000001,93338.21059315329,93347.0,2030,-8.740190525595152
+E06000005,106842.0,106342.0,2030,500.0
+E06000047,542479.5569257473,542501.0,2030,-21.29265255565681
+S12000033,255429.85009892532,255439.0,2030,-9.032867630937362
+S12000034,296583.51062325604,296590.0,2030,-6.40342012162319
+S12000041,120006.58397354413,120012.0,2030,-5.377240477640973
+W06000011,256211.5158715614,256238.6573629998,2030,-26.86628495283109
+W06000016,244011.4463617535,244037.48446900002,2030,-25.841607820738176
+W06000018,182996.6716557405,183019.33258800002,2030,-22.53800174334075
+E06000001,93308.26179563443,93317.0,2031,-8.68808073376054
+E06000005,106788.0,106288.0,2031,500.0
+E06000047,543594.6627413626,543616.0,2031,-21.188169081534795
+S12000033,256977.91206625625,256987.0,2031,-8.971962747209384
+S12000034,298520.55461874924,298527.0,2031,-6.361496936255415
+S12000041,120171.61512354168,120177.0,2031,-5.347832272161373
+W06000011,257051.4278473317,257078.38503200002,2031,-26.690396135571014
+W06000016,244353.5297655939,244379.41034200002,2031,-25.688822333185808
+W06000018,182971.960003998,182994.49773800003,2031,-22.419985159549366
+E06000001,93289.31287475835,93298.0,2032,-8.636889640529574
+E06000005,106739.0,106239.0,2032,500.0
+E06000047,544540.7741261601,544562.0,2032,-21.086108895352627
+S12000033,258477.97534992016,258487.0,2032,-8.91312089035617
+S12000034,300387.5985797926,300394.0,2032,-6.321737352776807
+S12000041,120316.64547535632,120322.0,2032,-5.319258815945474
+W06000011,257866.56051785412,257893.33833199987,2032,-26.51812449230608
+W06000016,244675.13645774196,244700.86179500003,2032,-25.540086983678922
+W06000018,182920.51794841894,182942.93437700009,2032,-22.303892531560244
+E06000001,93276.36350965305,93285.0,2033,-8.585588533887316
+E06000005,106706.0,106206.0,2033,500.0
+E06000047,545360.8813165369,545382.0,2033,-20.98455588608076
+S12000033,259954.03567086175,259963.0,2033,-8.853572690819801
+S12000034,302194.6400991293,302201.0,2033,-6.281847230529669
+S12000041,120443.67488972461,120449.0,2033,-5.289312338081677
+W06000011,258658.5753996861,258685.17770800012,2033,-26.344717055263324
+W06000016,244977.71859284124,245003.29293399997,2033,-25.391483771999496
+W06000018,182843.5708780414,182865.86810700016,2033,-22.187650808617352
+E06000001,93264.41472105912,93273.0,2034,-8.535928945662942
+E06000005,106689.0,106189.0,2034,500.0
+E06000047,546129.9850416954,546151.0,2034,-20.887524027599106
+S12000033,261396.09701004147,261405.0,2034,-8.79321144655718
+S12000034,303960.68130997795,303967.0,2034,-6.242314547363674
+S12000041,120550.70575487825,120556.0,2034,-5.260583015879559
+W06000011,259425.64127279824,259452.066808,2034,-26.174966391771058
+W06000016,245264.8135542332,245290.23742999995,2034,-25.24723728920025
+W06000018,182741.04890876106,182763.22681,2034,-22.075930900216537
+E06000001,93258.46383443936,93267.0,2035,-8.487284366535674
+E06000005,106661.0,106161.0,2035,500.0
+E06000047,546844.0843642717,546865.0,2035,-20.794194113638717
+S12000033,262818.1586538158,262827.0,2035,-8.734284498646312
+S12000034,305649.72286939534,305656.0,2035,-6.205035681664486
+S12000041,120632.73560764414,120638.0,2035,-5.2340401117794135
+W06000011,260167.8811889249,260194.13369300019,2035,-26.00902193121045
+W06000016,245535.76407536422,245561.041806,2035,-25.10729605355477
+W06000018,182614.28430048528,182636.347583,2035,-21.966959461335115
+E06000001,93237.51384994453,93246.0,2036,-8.438002292581471
+E06000005,106639.0,106139.0,2036,500.0
+E06000047,547516.1794572019,547537.0,2036,-20.699673862938187
+S12000033,264144.2213544719,264153.0,2036,-8.677471499442657
+S12000034,307270.7619287489,307277.0,2036,-6.168399898385512
+S12000041,120714.76217364125,120720.0,2036,-5.207823722200922
+W06000011,260887.47952813728,260913.5630960001,2036,-25.844614525772748
+W06000016,245786.4760651203,245811.6115680002,2036,-24.968737850719442
+W06000018,182463.26275958482,182485.21419299996,2036,-21.85972939083272
+E06000001,93225.56231571113,93234.0,2037,-8.390993330856729
+E06000005,106611.0,106111.0,2037,500.0
+E06000047,548186.2742086259,548207.0,2037,-20.610174079836465
+S12000033,265433.27989483194,265442.0,2037,-8.623315052243695
+S12000034,308840.7999574334,308847.0,2037,-6.134498005315094
+S12000041,120755.79018150562,120761.0,2037,-5.1827677316199745
+W06000011,261585.3012469314,261611.21755800015,2037,-25.68405960777813
+W06000016,246019.9515848764,246044.94657999984,2037,-24.83404273856811
+W06000018,182291.8746777689,182313.71649100003,2037,-21.756596254362165
+E06000001,93189.61148748237,93198.0,2038,-8.345755123595486
+E06000005,106590.0,106090.0,2038,500.0
+E06000047,548844.3643078483,548865.0,2038,-20.52145364113243
+S12000033,266722.3345250147,266731.0,2038,-8.568623109029948
+S12000034,310378.83482958324,310385.0,2038,-6.100839361779118
+S12000041,120768.81645071365,120774.0,2038,-5.157264851774342
+W06000011,262260.4149089893,262286.1677840001,2038,-25.528507111395825
+W06000016,246242.0686897043,246266.92766300013,2038,-24.703420726688428
+W06000018,182103.8958667907,182125.6326220001,2038,-21.655027655994193
+E06000001,93167.65546387646,93176.0,2039,-8.303267837858941
+E06000005,106573.0,106073.0,2039,500.0
+E06000047,549499.45328414,549520.0,2039,-20.43703516091012
+S12000033,267954.39151973056,267963.0,2039,-8.516084327849223
+S12000034,311880.8695159833,311887.0,2039,-6.0680912013395245
+S12000041,120793.84144620161,120799.0,2039,-5.1325348567383084
+W06000011,262918.0798791575,262943.674895,2039,-25.379756603164683
+W06000016,246456.0867909237,246480.814163,2039,-24.576639056839227
+W06000018,181902.34444540893,181923.978078,2039,-21.55763090422031

tests/test_all.py

@@ -20,7 +20,10 @@ def setUp(self):
     self.mye = MYEData.MYEData("./tests/raw_data")    
     self.npp = NPPData.NPPData("./tests/raw_data")    
     self.snpp = SNPPData.SNPPData("./tests/raw_data")    
-    self.snhp = SNHPData.SNHPData("./tests/raw_data")    
+    self.snhp = SNHPData.SNHPData("./tests/raw_data")
+
+    # fix issue with test dataset
+    self.snpp.data[utils.EN].PROJECTED_YEAR_NAME = self.snpp.data[utils.EN].PROJECTED_YEAR_NAME.astype(int)
 
     if not self.npp.data_api.key == "DUMMY" or not self.snpp.data_api.key == "DUMMY":
       print("Test requires NOMIS_API_KEY=DUMMY in env")
@@ -55,6 +58,23 @@ def test_utils(self):
     codes = 'A06000001'
     self.assertTrue(utils.country(codes) == [])
 
+    # naively, each element would be rounded down, making the total 10
+    fractional = np.array([0.1, 0.2, 0.3, 0.4]) * 11
+    integral = utils.integerise(fractional)
+    self.assertTrue(np.array_equal(integral, [1, 2, 3, 5]))
+
+    # 1.51 is NOT increased because 4.5 has a larger fractional part when total is rescaled to 17 from 16.91
+    fractional = np.array([1.1, 3.9, 4.5, 5.9, 1.51])
+    integral = utils.integerise(fractional)
+    self.assertTrue(np.array_equal(integral, [1, 4, 5, 6, 1]))
+
+    # another example that preserves sum
+    fractional = np.array([1.01] * 100)
+    integral = utils.integerise(fractional)
+    self.assertTrue(sum(integral) == 1.01 * 100)
+    self.assertTrue(np.array_equal(np.unique(integral), [1, 2]))
+
+
   def test_mye(self):
     self.assertEqual(self.mye.min_year(), 1991)
     self.assertEqual(self.mye.max_year(), 2016) # for test data, real data is 2039
@@ -174,6 +194,13 @@ def test_snpp_variant(self):
     # TODO more testing of results
     self.assertTrue(np.array_equal(base.OBS_VALUE, ppp.OBS_VALUE))
 
+  def test_snpp_custom_variant(self):
+    custom = self.snpp.custom_variant("./tests/raw_data/test_scenario.csv")
+
+    self.assertTrue(np.array_equal(custom.GEOGRAPHY_CODE.unique(), ['E06000005', 'E06000047', 'E06000001', 'S12000033', 'S12000034', 'S12000041', 'W06000011', 'W06000016', 'W06000018']))
+    self.assertTrue(np.array_equal(custom.PROJECTED_YEAR_NAME.unique(), [2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027]))
+    self.assertTrue(len(custom), 9 * 12 * 2 * 91) # 9 geogs, 12 years, 2 genders, 91 ages = 19656
+
   # test datasets have consistent ranges
   def test_consistency(self):
     self.npp.force_load_variants(["hhh", "ppp", "lll"])