From c5e5b51f9c0f6b4a98a41a2f6199479cac689a0f Mon Sep 17 00:00:00 2001 From: Chris <30196510+systemcatch@users.noreply.github.com> Date: Fri, 27 Jul 2018 13:39:35 +0100 Subject: [PATCH] Update key mapping for US-SPP (#1530) * Update key mapping for US-SPP * Create new mock data and update tests for SPP --- parsers/US_SPP.py | 7 +++---- parsers/test/mocks/US_SPP_Gen_Mix.pkl | Bin 4108 -> 3830 bytes parsers/test/test_US_SPP.py | 8 ++------ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/parsers/US_SPP.py b/parsers/US_SPP.py index 9a3cba76a2..4d85d4c620 100644 --- a/parsers/US_SPP.py +++ b/parsers/US_SPP.py @@ -18,7 +18,8 @@ 'Solar': 'solar', 'Natural Gas': 'gas', 'Diesel Fuel Oil': 'oil', - 'Waste Disposal Services': 'biomass' + 'Waste Disposal Services': 'biomass', + 'Coal': 'coal' } TIE_MAPPING = {'US-MISO->US-SPP': ['AMRN', 'DPC', 'GRE', 'MDU', 'MEC', 'NSP', 'OTP']} @@ -50,8 +51,7 @@ def data_processor(df, logger): # Remove leading whitespace in column headers. df.columns = df.columns.str.strip() - keys_to_remove = {'Coal Market', 'Coal Self', 'GMT MKT Interval', 'Average Actual Load', - 'Other', 'Waste Heat'} + keys_to_remove = {'GMT MKT Interval', 'Average Actual Load', 'Other', 'Waste Heat'} # Check for new generation columns. known_keys = MAPPING.keys() | keys_to_remove @@ -68,7 +68,6 @@ def data_processor(df, logger): processed_data = [] for index, row in df.iterrows(): production = row.to_dict() - production['coal'] = production['Coal Market'] + production['Coal Self'] extra_unknowns = sum([production[k] for k in unknown_keys]) production['unknown'] = production['Other'] + production['Waste Heat'] + extra_unknowns diff --git a/parsers/test/mocks/US_SPP_Gen_Mix.pkl b/parsers/test/mocks/US_SPP_Gen_Mix.pkl index 27cb36f5f4347350d966cd7150a2a67fef63f3b4..ba3f3baae674d150be39d090ba8b13ed022c5ec9 100644 GIT binary patch literal 3830 zcmdT{Yitx%6z<}-P$(}!pfS+_v2NAYoo;zGMlLP1)Gb?WD=4vSXSY*kbsuFONW~^4 z1e7p_K&Q}cp~fE)NtB=@goHK_g5lxPuF*fR8$v>O7>Py`LJWy|?mc&DZKvUn{$PIW znS0K6zI)Gi@3i-HIJ;psg?{in9eBO zY@e|c^iZao`F2~QO3{sUZ|bNWu^-A}pXr1pwpw0?nXT!Ps)K6O6$95?#wmkH${Z!1 z>aIMUQGJSc?oLaYZ%$|#rSq$!bk3qWm*1I~m329hNYpe95_(|C-a+DlM>{=8THrZ6 zi2jc0Lxke;mZLTGAr`-N6z$_pzT6(1Z$S^vQ`3uimwVBFq8In+EVmbU z{5&70Iqv2u%=_&W0g?#FQfw@dmNUrh~(r>v;@+Sp6+7zJ3?h z?-}3HbC5{9FfJY?7|tJ$KZDD}GnVNO^Jn5r@3;&1X~a5NoWyax5&k`G<=@vH{$2UF z?LUBg>$&}FKklQpAJ0eKe&o;M{`(fjmvWb5psRh#ZzC07UU*WY+PMGl@5j=i@mMYjB0AiljiMV{P# z=F+C>6nS^g*?ZU5rby>J=h{jwMVf9V7u@Jgku#S*UsZ4@MZlcc{!NNRZa+A3#e?WZSuPL&vt77-v!Zf-2=E-Lo%hP23F;$6Ir`f#Ey_F_80Pr-V8Q-3bg4bM)E>Ra9&M|MXs-#l?YPA< zTTQ@ikG9oBEVI=_EVI=F-1gkInuujw6MByw@5EH^syO*L{k0jnR{ABd>+^JFCUif_ ztg%H-flD>}D3n6wMdc;sWu-_dR$fuuil~$*9;VFV%;kPu76)fhT+l`Y;8xGOfk!8l zj!cl-p$Bmni>1XPYZjdn^mUe!XXXTGAE-gg=z==TNs*1JA3U`02^b;h-=egq(ZbN& zK=TDnSsr#?W)AdI2IW{~PysCE0By=JsPrgrJoM&yr#|#X4B6124%6v}b`JVj+B9NM IYXpk_0hxq@=Kufz literal 4108 zcmdT{eNYr-7zYKjI`Jb6HLFdFAmzL}#E;A3l|zv300YWu#;)a9TmD|lZCFPlAnNidg6e7T^h7iaPcpGBWv(vlz5R%=OWj9aO! zfT0o?VzgMdZfNSnvn9%WiXi!TudJ!246omFUoP+CmkW|cm6Dc$)TH735@Dqvn~QlF zX1>Jj6CQBq_`nAN;utkH@=SB^>FMREF>GF6puD0IGcON#l`>wE_)3`9EeRgKPgbOW zM*$f%mJ&7fB3LP}&xa`~_F4tMa<>>2Q!FwSM&f(|1TrMx&Cn&HqGz$f9siuzO zGRp*6@N!uJxaF4teH6F2vPAOhae2HFkRUXhhboTd@&X>Oz)L!%(C^hBCUHx7SrI@) zuJFsiEQC4BJOa>Txus>k5>367Q5Fk4Og@UsS47wZHJ)3zLXcoXxP=}i0Ff>~5A3)s zZ(t?o;464fnc|l;NVqr)yYQNCGo!H`hFGYc=jH}$_g(jP&G9MGOW;?(c)#W zW3u9h=LD)LK*7WWaf-eN;?&3vn4AegO`N8VC_Sg<3(l!|lwR|U#?etxsrt3Km3(*r zuC=Y?wKZ_HF?)S0Db=rMTM6YU40kc!m#u_;0SRpwZ)TXou(A!~wleHu*vZDXF>GL% zy$N+H8NSZ2f#H@-m|x!}n%BmPE}Z{?F6!@D(M97`FQN-3)oOx`*ZQ9qJ^`!8o1! zf4k^=V5`RZFRI3UO|8cLBGrVhXF)Z-Fa6B^x(3H*2hpDxWN|_Ca|~w&Q8zh=I+@H* zn14RQfm-C%K8NUj5%oTM1@#ZtVf_9&^dDq6tAmdBPed$D?7(<)2l6U9Xg;o~9W=h` z9<&dTJ#@VN!#$YqO?xq}e=q92vJdAywU5S;`5{C#MC0V+AsT10b<(&VODE3rM+oh& zkY9C8C+4vYQ6B#;+PfMM_ch?Ub~4;CWYc}=ILE)c0rP#2#qVMG4#S8pV3dA=F4YM& z;QD}1{|DRIcmzQ=!oMKW!T)xn|DiRz(*K*vzm1-T^-i6J>pyS}L4_ed{SU%;#Q#qb z*E0?4yJiS6e)^z4V&gp1Sw7Qoe@cfiGEVvTwqX3B7UUmeHevtg6D_!&0p?%Vf_kSn zBL5ABS2CQj0mnD5N8R-GgwFG0Guj_Dqg~sKyc6pXV;R2LgyTs~IPdcetxd@Lp1ns@ z^#66Ebw8bf>mv@VM^6UU5&ZN!N^}g4M{K(Pz@y*QfeehB>=?8uub=sW7qRJl?QFg% zrd!4A(2zfp7rqxmdC__Abdc8fopW1lW0TNfs>_`*qLTbS%`TD0Sx zePQC>onw0JbeP249+z@BxtH8Ae|z_wrM+bFu~qR`RrZp&lx_80YkNtZZ^GkTeJ}a= z=fdOl{k`PIqxsJ)HTO~77kz!izP)?J#)duuu=nb3`ba|0M?bg69VHr^RL(!>L`M!f z!w;8Y;@=!C<82n}ycCNq#g=w&iaFQGDo^3I98?)1l*=a^>D+n8CgWH%5 zIi8=O)owLr+l<*ZV|J=B+ps29yJ1bhZLDor6RX{@CN|@mfZIshW;d(}xQ*F{HL=-Q zP4u(*1>UOR$7j(w{H>ygXUERVPkGyl_Bu#ycF9lAeU(nE4^K+*ElVaNlcwJO?E)8R zo@iP*@?;h{9DJdDRZ|AJx#s@HDS;W}$uk85o)4CgQ@_4?^NvOrdBpcc?!g-ih^xk> z9=l`_xx0F1V=%6eqx7y0CaS2xV~Wgh*a zX!|f$91k5Kv=(Cg`L!@XUvZt@mWelrGsKx-L58@)KSHHcPwj=CagpqUA|m$gGcZ9iUVcP z&HD6iPWD1u2dyZysAI4XRYyU08PGMd0(uGR#YVkF#X+%Iymin!#u+#0l>$OQhDE3s QyUPWIrw#NnEl_O!3))sXumAu6 diff --git a/parsers/test/test_US_SPP.py b/parsers/test/test_US_SPP.py index 2ec6d9ca3e..a508564ac1 100644 --- a/parsers/test/test_US_SPP.py +++ b/parsers/test/test_US_SPP.py @@ -32,16 +32,12 @@ def test_fetch_production(self): with self.subTest(): self.assertEqual(len(data), 23) - # Multiple coal keys in data source, ensure they are summed correctly. - with self.subTest(): - self.assertEqual(round(datapoint['production']['coal'], 2), 15963.60) - # Unknown keys must be assigned and summed. with self.subTest(): - self.assertEqual(round(datapoint['production']['unknown'], 2), 63.67) + self.assertEqual(round(datapoint['production']['unknown'], 2), 33.1) with self.subTest(): - expected_dt = get(datetime(2018, 2, 23, 12, 25), 'UTC').datetime + expected_dt = get(datetime(2018, 7, 27, 11, 45), 'UTC').datetime self.assertEqual(datapoint['datetime'], expected_dt) with self.subTest():