From e7cc440058009d30aabe623503423a591a4e8dff Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:16:36 +0100 Subject: [PATCH 1/7] Show the constituency of the petition creator for archived petitions --- .../archived/petitions/_petition_details.html.erb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/views/admin/archived/petitions/_petition_details.html.erb b/app/views/admin/archived/petitions/_petition_details.html.erb index a7f53047f..464fbb411 100644 --- a/app/views/admin/archived/petitions/_petition_details.html.erb +++ b/app/views/admin/archived/petitions/_petition_details.html.erb @@ -8,12 +8,20 @@ <% if @petition.anonymized? %> <dt>Anonymized</dt> <dd><%= date_time_format(@petition.anonymized_at) %></dd> - <% elsif @petition.creator %> + <% elsif creator = @petition.creator %> <dt>Creator</dt> <dd> - <%= @petition.creator.name %><br /> - <%= auto_link(@petition.creator.email) %> + <%= creator.name %><br /> + <%= auto_link(creator.email) %> </dd> + + <% if constituency = creator.constituency %> + <dt>Constituency</dt> + <dd> + <span class="creator-constituency"><%= constituency.name %></span><br /> + <small class="creator-constituency-region"><%= constituency.region.name %></small> + </dd> + <% end %> <% end %> <% if @petition.removed? %> From 1e6e8ad0a227d77b0e9921724b5b74083e88f293 Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:17:35 +0100 Subject: [PATCH 2/7] Add example postcodes for the new constituencies Sourced from the following postcode to WPC lookup table: https://geoportal.statistics.gov.uk/datasets/f60c78533aa7462cb934bb4a81afc1e0/about Generated using the following query: SELECT pconcd AS ons_code, REPLACE(pcd, ' ', '') AS example_postcode FROM ( SELECT pconcd, first_value(pcd) OVER ( PARTITION BY pconcd ORDER BY random() ) AS pcd FROM postcodes WHERE pconcd IS NOT NULL ) AS p GROUP BY ons_code, example_postcode ORDER BY ons_code; --- data/example_postcodes.yml | 645 +++++++++++++++++++++++++++++++++++++ 1 file changed, 645 insertions(+) diff --git a/data/example_postcodes.yml b/data/example_postcodes.yml index 349443853..9732a7a88 100644 --- a/data/example_postcodes.yml +++ b/data/example_postcodes.yml @@ -532,6 +532,567 @@ E14001059: M230BX E14001060: TA188AE E14001061: YO104PB E14001062: YO329LL +E14001063: GU147GR +E14001064: WS98UZ +E14001065: M337FX +E14001066: DE552ZY +E14001067: RH201FL +E14001068: NG166GW +E14001069: TN235TA +E14001070: M345NH +E14001071: HP218HL +E14001072: OX165YA +E14001073: RM95AE +E14001074: S714YQ +E14001075: S738SQ +E14001076: LA127DA +E14001077: SS142RT +E14001078: RG218UZ +E14001079: S817JD +E14001080: BA10GN +E14001081: SW128AG +E14001082: HP91SR +E14001083: BR40PD +E14001084: MK409DR +E14001085: SE19WS +E14001086: E13NS +E14001087: HU192YS +E14001088: TN316RN +E14001089: DA83JA +E14001090: OX201NJ +E14001091: L439WY +E14001092: B152YR +E14001093: B235DF +E14001094: B147RS +E14001095: B330YR +E14001096: B129TY +E14001097: B319EN +E14001098: B203DF +E14001099: B139WA +E14001100: B261AD +E14001101: DL140LS +E14001102: BB24JP +E14001103: M109HN +E14001104: FY52LW +E14001105: FY39XL +E14001106: NE391QB +E14001107: NE241PW +E14001108: BN176HG +E14001109: S434GD +E14001110: BL25NA +E14001111: BL33HG +E14001112: BL65JH +E14001113: L229RA +E14001114: PE217TS +E14001115: BH80HP +E14001116: BH119AB +E14001117: RG121DN +E14001118: BD30LD +E14001119: BD62BN +E14001120: BD13RN +E14001121: CM73XF +E14001122: NW103QR +E14001123: HA99RL +E14001124: TW27DT +E14001125: CM166AR +E14001126: TA65PR +E14001127: YO433LL +E14001128: DN208SP +E14001129: BN107JP +E14001130: BN24WJ +E14001131: BS29LE +E14001132: BS16EB +E14001133: BS151RA +E14001134: BS107NT +E14001135: BS149JR +E14001136: NR219RZ +E14001137: BR14JH +E14001138: B380EE +E14001139: EN110FD +E14001140: NG93DQ +E14001141: HP224HH +E14001142: BB113PR +E14001143: DE143TD +E14001144: BL83LY +E14001145: M269RG +E14001146: IP327DX +E14001147: HX38FR +E14001148: TR60EG +E14001149: CB22DT +E14001150: WS151LZ +E14001151: CT45SB +E14001152: CA13JQ +E14001153: SM69WJ +E14001154: SS72PZ +E14001155: EX55AQ +E14001156: IP60SA +E14001157: ME206NB +E14001158: SK72EE +E14001159: CM11DL +E14001160: SW35SX +E14001161: GL515DD +E14001162: HP79QX +E14001163: CH49FA +E14001164: CW57PD +E14001165: S404PS +E14001166: PO201AD +E14001167: E47PR +E14001168: SN139FJ +E14001169: EN54XP +E14001170: PR69FN +E14001171: BH220HR +E14001172: SW71BL +E14001173: DH19GN +E14001174: CO168PB +E14001175: SW83JF +E14001176: CO33QX +E14001177: HD72HG +E14001178: CW124BX +E14001179: NN189FJ +E14001180: CV22AD +E14001181: CV57RH +E14001182: CV47BX +E14001183: NE126LX +E14001184: RH109GG +E14001185: CW28FT +E14001186: CR07EB +E14001187: CR27YA +E14001188: CR03RT +E14001189: RM125TJ +E14001190: DL56QQ +E14001191: DA26QP +E14001192: NN116LP +E14001193: DE37DY +E14001194: DE249QN +E14001195: DE655HL +E14001196: WF176EG +E14001197: OX129HP +E14001198: DN12JU +E14001199: DN110QQ +E14001200: DN68SP +E14001201: RH28HP +E14001202: CT144DA +E14001203: WR114SR +E14001204: DY12RF +E14001205: SE249BD +E14001206: LU55YW +E14001207: NW106QA +E14001208: UB55SF +E14001209: W72AB +E14001210: RG53HB +E14001211: SR79RY +E14001212: RH199RA +E14001213: E66EH +E14001214: GU345XP +E14001215: RH89LE +E14001216: CT119JN +E14001217: SN95RB +E14001218: BN150AL +E14001219: BN228HF +E14001220: SO55EA +E14001221: N182JL +E14001222: CH653EY +E14001223: SE94DQ +E14001224: CB63UG +E14001225: EN35EJ +E14001226: CM167RX +E14001227: KT174NN +E14001228: NG104WZ +E14001229: SE20PA +E14001230: KT108HT +E14001231: EX49AH +E14001232: EX30EH +E14001233: PO175GX +E14001234: GU99YJ +E14001235: ME172NT +E14001236: TW140AN +E14001237: BS128DY +E14001238: NW23YZ +E14001239: CT202QD +E14001240: GL181BS +E14001241: BS185SJ +E14001242: FY45QU +E14001243: DN211GU +E14001244: NE165UB +E14001245: NG57WE +E14001246: ME87UQ +E14001247: BA98AT +E14001248: GL25EJ +E14001249: GU68QY +E14001250: DN149RH +E14001251: M340BH +E14001252: PO123LT +E14001253: NG310SJ +E14001254: DA124BX +E14001255: DN329UR +E14001256: NR310BD +E14001257: SE186TJ +E14001258: KT245AZ +E14001259: N168GL +E14001260: E84AZ +E14001261: B622EU +E14001262: HX28WJ +E14001263: SO38BZ +E14001264: W140FX +E14001265: NW61QZ +E14001266: LE189BL +E14001267: CM201EL +E14001268: HP42JQ +E14001269: HG12DG +E14001270: HA74WR +E14001271: HA33QY +E14001272: TS260DD +E14001273: CO111UA +E14001274: TN380WX +E14001275: PO110HX +E14001276: UB39AR +E14001277: SK61SB +E14001278: HP24NW +E14001279: NW94BL +E14001280: RG99BP +E14001281: HR99DP +E14001282: CT70EG +E14001283: CM233AQ +E14001284: WD234WQ +E14001285: NE463PY +E14001286: M243UY +E14001287: SK139LZ +E14001288: LE99JQ +E14001289: SG49BW +E14001290: NW1W9XR +E14001291: EX122NN +E14001292: RM124DG +E14001293: N87AU +E14001294: RH149BF +E14001295: DH59FY +E14001296: BN42BJ +E14001297: HD58QD +E14001298: PE174UR +E14001299: BB55FU +E14001300: RM52DP +E14001301: IG18XX +E14001302: IP20RQ +E14001303: PO381NR +E14001304: PO409TN +E14001305: N59FB +E14001306: W1A3WD +E14001307: NE312UG +E14001308: BD219FY +E14001309: CV330FD +E14001310: SW59QT +E14001311: NN141UE +E14001312: KT65PJ +E14001313: HU80HP +E14001314: HU54NB +E14001315: HU106RY +E14001316: DY67LA +E14001317: L286AA +E14001318: LA29BG +E14001319: LS63HL +E14001320: LS158DX +E14001321: LS178BB +E14001322: LS184EB +E14001323: LS116DE +E14001324: WF31LP +E14001325: LS134UB +E14001326: LE54QA +E14001327: LE22YQ +E14001328: LE28TS +E14001329: M297PJ +E14001330: BN72LP +E14001331: SE69NG +E14001332: SE136JU +E14001333: SE231XU +E14001334: E152AY +E14001335: DE137AS +E14001336: LN55JH +E14001337: L242TR +E14001338: L53LN +E14001339: L96DJ +E14001340: L179WA +E14001341: L280QN +E14001342: LE119JD +E14001343: LN121PZ +E14001344: NR321QT +E14001345: LU40XB +E14001346: LU31XU +E14001347: SK110HY +E14001348: RG424HY +E14001349: ME168FW +E14001350: WN49NP +E14001351: CM28LP +E14001352: M610EN +E14001353: M139GE +E14001354: M146WT +E14001355: NG196AZ +E14001356: SN104BF +E14001357: LE72JX +E14001358: B939EP +E14001359: MK430EL +E14001360: OX270AH +E14001361: CW95FW +E14001362: DE76HP +E14001363: BH212BH +E14001364: LE44NH +E14001365: PE378EY +E14001366: BN68DG +E14001367: TS57QZ +E14001368: TS89QU +E14001369: MK78AX +E14001370: MK168WZ +E14001371: SM44DR +E14001372: LA33AR +E14001373: SO44WG +E14001374: SO410AL +E14001375: NG241GE +E14001376: RG168HB +E14001377: NE17RQ +E14001378: NE77YN +E14001379: NE35JG +E14001380: ST50DF +E14001381: TQ125XH +E14001382: DL167GA +E14001383: S729AR +E14001384: SG191WD +E14001385: PL276NU +E14001386: GL510XZ +E14001387: EX379DN +E14001388: BH215QB +E14001389: DH96DL +E14001390: PE135DF +E14001391: S211JQ +E14001392: RG291FJ +E14001393: SG99PB +E14001394: BS308JJ +E14001395: HR60JN +E14001396: NR117NZ +E14001397: NE716XP +E14001398: SY109YH +E14001399: BS192TQ +E14001400: CV78JQ +E14001401: PE262UG +E14001402: CM63BN +E14001403: RG209DL +E14001404: LE651HP +E14001405: NR219PU +E14001406: NN38ZW +E14001407: NN40EG +E14001408: NR70WZ +E14001409: NR21ET +E14001410: NG12JA +E14001411: NG161EN +E14001412: NG21PN +E14001413: CV114JX +E14001414: DA162AG +E14001415: OL41JT +E14001416: OL90LA +E14001417: BR52QL +E14001418: WF44PY +E14001419: OX44NT +E14001420: OX12HL +E14001421: SE159JH +E14001422: BB87GN +E14001423: S753SA +E14001424: CA139PR +E14001425: PE67SL +E14001426: PL65FR +E14001427: PL48YH +E14001428: WF84HT +E14001429: BH136DR +E14001430: E33RF +E14001431: PO29QH +E14001432: PO13NJ +E14001433: PR23AT +E14001434: SW153WU +E14001435: NW69RB +E14001436: S627NR +E14001437: SS43JH +E14001438: RG11HF +E14001439: RG317YU +E14001440: TS79DL +E14001441: B975UH +E14001442: RH27HT +E14001443: PR38AQ +E14001444: DL94AR +E14001445: SW133AX +E14001446: OL164DS +E14001447: ME11RP +E14001448: RM124DL +E14001449: SO517JH +E14001450: BB31SA +E14001451: S818NR +E14001452: S652LH +E14001453: CV219FZ +E14001454: HA63RE +E14001455: WA45QU +E14001456: KT138NX +E14001457: NG116HJ +E14001458: LE150SS +E14001459: M52YX +E14001460: SP52PR +E14001461: YO112AD +E14001462: DN161EY +E14001463: L319AB +E14001464: LS256DZ +E14001465: TN131TY +E14001466: S39PJ +E14001467: S102JN +E14001468: S117GE +E14001469: S88LW +E14001470: S981RN +E14001471: NG150BY +E14001472: BD181NT +E14001473: SY19ET +E14001474: ME122FD +E14001475: BD235NH +E14001476: LN41EL +E14001477: SL13JS +E14001478: B676EH +E14001479: B913QL +E14001480: SS170PU +E14001481: CB37JT +E14001482: SN169XA +E14001483: DE738HE +E14001484: PL210TT +E14001485: DT40PJ +E14001486: PL143NZ +E14001487: PE111YY +E14001488: LE174SH +E14001489: NR93ES +E14001490: NN97RE +E14001491: L402QG +E14001492: NE348RA +E14001493: SY81GN +E14001494: IP300NL +E14001495: PL73YU +E14001496: WD31BU +E14001497: PE389UP +E14001498: BA148JL +E14001499: SO151DJ +E14001500: SO150EP +E14001501: SS11FS +E14001502: SS228NF +E14001503: EN40EY +E14001504: PR86BA +E14001505: TW184DN +E14001506: WF149DG +E14001507: AL13FU +E14001508: TR79EP +E14001509: WN57QR +E14001510: WA91HW +E14001511: TR262FW +E14001512: PE199FR +E14001513: ST174XJ +E14001514: ST68TT +E14001515: SK148LJ +E14001516: AL69SZ +E14001517: SK26HA +E14001518: TS183DG +E14001519: TS159NR +E14001520: ST15JT +E14001521: ST68EX +E14001522: ST39EA +E14001523: ST195AA +E14001524: DY98NR +E14001525: E34UW +E14001526: CV364QJ +E14001527: SW23ZN +E14001528: M311AD +E14001529: GL53BY +E14001530: IP112XH +E14001531: SR52AQ +E14001532: GU153FF +E14001533: TN63PF +E14001534: SM38ST +E14001535: B761QN +E14001536: SN253JG +E14001537: SN36DA +E14001538: B755SX +E14001539: SK93FA +E14001540: TA28LA +E14001541: TF22DL +E14001542: GL29GL +E14001543: TF92TW +E14001544: YO170XQ +E14001545: BS378RN +E14001546: RM175DP +E14001547: DY47YX +E14001548: TA44SE +E14001549: TN85PL +E14001550: SW170AN +E14001551: TQ26QQ +E14001552: EX392HU +E14001553: N41HG +E14001554: TR37GA +E14001555: TN30DD +E14001556: TW118PD +E14001557: NE280YU +E14001558: UB111FH +E14001559: SW81PD +E14001560: WF13JX +E14001561: L458LF +E14001562: WS13XY +E14001563: E173DA +E14001564: WA28EE +E14001565: WA51XL +E14001566: CV325WN +E14001567: SR53DN +E14001568: WD17RJ +E14001569: NR352TX +E14001570: ME185HG +E14001571: NN109BS +E14001572: BA51DA +E14001573: AL73SJ +E14001574: B706EB +E14001575: DT66NZ +E14001576: E66WD +E14001577: L395WE +E14001578: CB88GJ +E14001579: WR141RN +E14001580: LA99AD +E14001581: BS231UB +E14001582: LS178JW +E14001583: CA143AT +E14001584: WA86SF +E14001585: WN67DH +E14001586: SW208XE +E14001587: SO321YD +E14001588: SL57DB +E14001589: CH601XN +E14001590: CM81EB +E14001591: OX87SS +E14001592: GU229EE +E14001593: RG114TT +E14001594: WV45DH +E14001595: WV12UZ +E14001596: WV60BE +E14001597: WR38BF +E14001598: M281UU +E14001599: BN132QF +E14001600: HP137FW +E14001601: DY116BD +E14001602: M339AP +E14001603: TA203HJ +E14001604: YO16XU +E14001605: YO195QS +N05000001: BT161XZ +N05000002: BT148QS +N05000003: BT68LY +N05000004: BT119QU +N05000005: BT401PP +N05000006: BT515JD +N05000007: BT750LP +N05000008: BT487SU +N05000009: BT179PH +N05000010: BT448JF +N05000011: BT359SB +N05000012: BT449BP +N05000013: BT196YX +N05000014: BT413BP +N05000015: BT341QH +N05000016: BT235YR +N05000017: BT324AZ +N05000018: BT785NT N06000001: BT54SG N06000002: BT153PH N06000003: BT68AJ @@ -609,6 +1170,58 @@ S14000056: G728DG S14000057: FK77AQ S14000058: AB38QG S14000059: G813BG +S14000060: AB166SL +S14000061: AB159YX +S14000062: AB420NG +S14000063: ML68LW +S14000064: FK109SQ +S14000065: PH139EG +S14000066: DD77AW +S14000067: PA238BH +S14000068: EH484NY +S14000069: KW147JF +S14000070: G698AD +S14000071: KY25TJ +S14000072: G674LT +S14000073: DG29AW +S14000074: EH446LN +S14000075: DD22NL +S14000076: KY127RG +S14000077: ML93DY +S14000078: EH68JW +S14000079: EH35JN +S14000080: EH166YE +S14000081: EH114EH +S14000082: EH113YZ +S14000083: FK65NQ +S14000084: G59AG +S14000085: G128NF +S14000086: G27YS +S14000087: G444XG +S14000088: G537WS +S14000089: G117PH +S14000090: KY83DB +S14000091: AB545XJ +S14000092: ML33EE +S14000093: PA153AL +S14000094: PH414QU +S14000095: EH548WF +S14000096: EH421PS +S14000097: G663EF +S14000098: IV301QZ +S14000099: ML20RG +S14000100: KY84SZ +S14000101: PA13DF +S14000102: PA91AE +S14000103: DD25BF +S14000104: G727RX +S14000105: FK94UD +S14000106: G811DU +S14000107: KA72AN +S14000108: TD90RA +S14000109: KA129BA +S14000110: KA182JH +S14000111: AB125GZ W07000041: LL689EN W07000042: CH71SH W07000043: LL129HL @@ -649,3 +1262,35 @@ W07000077: NP16LU W07000078: CF66BH W07000079: CF52HG W07000080: CF245JX +W07000081: CF340DT +W07000082: CH73PR +W07000083: LL228YE +W07000084: CF89QP +W07000085: LD37PP +W07000086: CF315BL +W07000087: SA335HE +W07000088: CF82BA +W07000089: CF38ED +W07000090: CF149EB +W07000091: CF15SY +W07000092: CF48AY +W07000093: SA625HB +W07000094: CH71EJ +W07000095: LL182YB +W07000096: LL414EB +W07000097: SA49HS +W07000098: SA152WD +W07000099: CF482BF +W07000100: SA678UR +W07000101: NP70BY +W07000102: SY219HU +W07000103: SA68DP +W07000104: NPT5PN +W07000105: NP100BP +W07000106: CF379DD +W07000107: CF425AB +W07000108: SA11SN +W07000109: NP444EF +W07000110: CF68YS +W07000111: LL112AN +W07000112: LL616YJ From b83cdc6bbc54d9679420c271e4133082773aeaac Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:28:14 +0100 Subject: [PATCH 3/7] Record the start and end date for constituencies Also adjust the uniqueness index on the slug to only apply to 'current' constituencies (i.e. records where the end_date is NULL). --- app/jobs/fetch_constituencies_job.rb | 4 +++- app/lib/feed/constituencies.rb | 4 +++- app/views/constituencies/index.json.jbuilder | 2 ++ ...240531160714_add_start_and_end_date_to_constituencies.rb | 6 ++++++ ...62307_change_constituency_slug_index_to_unique_active.rb | 6 ++++++ db/schema.rb | 6 ++++-- spec/fixtures/files/constituencies.xml | 6 ++++++ 7 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20240531160714_add_start_and_end_date_to_constituencies.rb create mode 100644 db/migrate/20240531162307_change_constituency_slug_index_to_unique_active.rb diff --git a/app/jobs/fetch_constituencies_job.rb b/app/jobs/fetch_constituencies_job.rb index 89d2d833e..f3219c819 100644 --- a/app/jobs/fetch_constituencies_job.rb +++ b/app/jobs/fetch_constituencies_job.rb @@ -4,13 +4,15 @@ class FetchConstituenciesJob < ApplicationJob end def perform - constituencies.each do |external_id, name, ons_code| + constituencies.each do |external_id, name, ons_code, start_date, end_date| begin Constituency.for(external_id) do |constituency| constituency.name = name constituency.ons_code = ons_code constituency.example_postcode = example_postcodes[ons_code] constituency.region_id = regions[external_id] + constituency.start_date = start_date + constituency.end_date = end_date if mp = mps[external_id] constituency.mp_id = mp.id diff --git a/app/lib/feed/constituencies.rb b/app/lib/feed/constituencies.rb index 3cc5c49ea..8e5368df6 100644 --- a/app/lib/feed/constituencies.rb +++ b/app/lib/feed/constituencies.rb @@ -4,10 +4,12 @@ class Constituency < Entry attribute :id, :string, ".//d:Constituency_Id" attribute :name, :string, ".//d:Name" attribute :ons_code, :string, ".//d:ONSCode" + attribute :start_date, :date, ".//d:StartDate" + attribute :end_date, :date, ".//d:EndDate" end self.model = "Constituencies" - self.columns = "Constituency_Id,Name,ONSCode" + self.columns = "Constituency_Id,Name,ONSCode,StartDate,EndDate" self.filter = "EndDate%20eq%20null" self.klass = Constituency end diff --git a/app/views/constituencies/index.json.jbuilder b/app/views/constituencies/index.json.jbuilder index 9b389db66..508c22cbd 100644 --- a/app/views/constituencies/index.json.jbuilder +++ b/app/views/constituencies/index.json.jbuilder @@ -5,6 +5,8 @@ json.cache! :constituencies, expires_in: 1.hour do json.party constituency.party json.constituency constituency.name json.ons_code constituency.ons_code + json.start_date constituency.start_date + json.end_date constituency.end_date end end end diff --git a/db/migrate/20240531160714_add_start_and_end_date_to_constituencies.rb b/db/migrate/20240531160714_add_start_and_end_date_to_constituencies.rb new file mode 100644 index 000000000..e77c7e8fc --- /dev/null +++ b/db/migrate/20240531160714_add_start_and_end_date_to_constituencies.rb @@ -0,0 +1,6 @@ +class AddStartAndEndDateToConstituencies < ActiveRecord::Migration[7.1] + def change + add_column :constituencies, :start_date, :date + add_column :constituencies, :end_date, :date + end +end diff --git a/db/migrate/20240531162307_change_constituency_slug_index_to_unique_active.rb b/db/migrate/20240531162307_change_constituency_slug_index_to_unique_active.rb new file mode 100644 index 000000000..8d086e0d7 --- /dev/null +++ b/db/migrate/20240531162307_change_constituency_slug_index_to_unique_active.rb @@ -0,0 +1,6 @@ +class ChangeConstituencySlugIndexToUniqueActive < ActiveRecord::Migration[7.1] + def change + remove_index :constituencies, :slug, unique: true + add_index :constituencies, :slug, unique: true, where: "end_date IS NULL" + end +end diff --git a/db/schema.rb b/db/schema.rb index 871f57cf5..470b12387 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_05_23_163632) do +ActiveRecord::Schema[7.1].define(version: 2024_05_31_162307) do # These are extensions that must be enabled in order to support this database enable_extension "intarray" enable_extension "plpgsql" @@ -263,9 +263,11 @@ t.string "example_postcode", limit: 30 t.string "party", limit: 100 t.string "region_id", limit: 30 + t.date "start_date" + t.date "end_date" t.index ["external_id"], name: "index_constituencies_on_external_id", unique: true t.index ["region_id"], name: "index_constituencies_on_region_id" - t.index ["slug"], name: "index_constituencies_on_slug", unique: true + t.index ["slug"], name: "index_constituencies_on_slug", unique: true, where: "(end_date IS NULL)" end create_table "constituency_petition_journals", id: :serial, force: :cascade do |t| diff --git a/spec/fixtures/files/constituencies.xml b/spec/fixtures/files/constituencies.xml index c420dc855..552ce31e7 100644 --- a/spec/fixtures/files/constituencies.xml +++ b/spec/fixtures/files/constituencies.xml @@ -18,6 +18,8 @@ <d:Constituency_Id m:type="Edm.Int32">3320</d:Constituency_Id> <d:Name>Bethnal Green and Bow</d:Name> <d:ONSCode>E14000555</d:ONSCode> + <d:StartDate m:type="Edm.DateTime">2010-04-13T00:00:00</d:StartDate> + <d:EndDate m:type="Edm.DateTime" m:null="true" /> </m:properties> </content> </entry> @@ -35,6 +37,8 @@ <d:Constituency_Id m:type="Edm.Int32">3427</d:Constituency_Id> <d:Name>Coventry North East</d:Name> <d:ONSCode>E14000649</d:ONSCode> + <d:StartDate m:type="Edm.DateTime">2010-04-13T00:00:00</d:StartDate> + <d:EndDate m:type="Edm.DateTime" m:null="true" /> </m:properties> </content> </entry> @@ -52,6 +56,8 @@ <d:Constituency_Id m:type="Edm.Int32">3724</d:Constituency_Id> <d:Name>Sheffield, Brightside and Hillsborough</d:Name> <d:ONSCode>E14000921</d:ONSCode> + <d:StartDate m:type="Edm.DateTime">2010-04-13T00:00:00</d:StartDate> + <d:EndDate m:type="Edm.DateTime" m:null="true" /> </m:properties> </content> </entry> From d85cd8b84fe6f169a3e6ad82b6b1cbcb4e99f084 Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:30:11 +0100 Subject: [PATCH 4/7] Only retry the once, not forever --- app/jobs/fetch_constituencies_job.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/jobs/fetch_constituencies_job.rb b/app/jobs/fetch_constituencies_job.rb index f3219c819..9e500a361 100644 --- a/app/jobs/fetch_constituencies_job.rb +++ b/app/jobs/fetch_constituencies_job.rb @@ -6,6 +6,8 @@ class FetchConstituenciesJob < ApplicationJob def perform constituencies.each do |external_id, name, ons_code, start_date, end_date| begin + retried = false + Constituency.for(external_id) do |constituency| constituency.name = name constituency.ons_code = ons_code @@ -29,7 +31,8 @@ def perform constituency.save! end rescue ActiveRecord::RecordNotUnique => e - retry + retry unless retried + retried = true end end end From a57c2cc41748ac4d6690dee53bb47fc29837ef0a Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:31:12 +0100 Subject: [PATCH 5/7] Raise an error if there are no lookup values If there's no lookup value then something has gone wrong. --- app/jobs/fetch_constituencies_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/fetch_constituencies_job.rb b/app/jobs/fetch_constituencies_job.rb index 9e500a361..48bacaf0c 100644 --- a/app/jobs/fetch_constituencies_job.rb +++ b/app/jobs/fetch_constituencies_job.rb @@ -11,8 +11,8 @@ def perform Constituency.for(external_id) do |constituency| constituency.name = name constituency.ons_code = ons_code - constituency.example_postcode = example_postcodes[ons_code] - constituency.region_id = regions[external_id] + constituency.example_postcode = example_postcodes.fetch(ons_code) + constituency.region_id = regions.fetch(external_id) constituency.start_date = start_date constituency.end_date = end_date From 5d800231cacf9fc86dcd55ec56e18056f76f1b8c Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:33:30 +0100 Subject: [PATCH 6/7] Update feeds to pull in old and new constituencies Order the results so the old constituencies have their end_date set first so the new constituencies don't raise a duplicate value error when they have the same slug as the old constituency. --- app/lib/feed.rb | 8 ++++++-- app/lib/feed/constituencies.rb | 3 ++- app/lib/feed/constituency_regions.rb | 2 +- app/lib/feed/departments.rb | 3 ++- app/lib/feed/members.rb | 3 ++- app/lib/feed/regions.rb | 3 ++- spec/jobs/fetch_constituencies_job_spec.rb | 6 +++--- spec/jobs/fetch_regions_job_spec.rb | 2 +- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/app/lib/feed.rb b/app/lib/feed.rb index a5d30989d..34bd2b22a 100644 --- a/app/lib/feed.rb +++ b/app/lib/feed.rb @@ -6,7 +6,7 @@ class Base with_options instance_writer: false do class_attribute :host, :path, :model - class_attribute :columns, :filter + class_attribute :columns, :filter, :orderby class_attribute :open_timeout, :timeout class_attribute :xpath, :klass end @@ -22,7 +22,7 @@ def url end def endpoint - "#{path}/#{model}?$select=#{columns}&$filter=#{filter}" + "#{path}/#{model}?$select=#{columns}&$filter=#{escape(filter)}&$orderby=#{orderby||columns}" end def each(&block) @@ -39,6 +39,10 @@ def size private + def escape(filter) + CGI.escape(filter) + end + def entries @entries ||= fetch_entries end diff --git a/app/lib/feed/constituencies.rb b/app/lib/feed/constituencies.rb index 8e5368df6..00b6bcaee 100644 --- a/app/lib/feed/constituencies.rb +++ b/app/lib/feed/constituencies.rb @@ -10,7 +10,8 @@ class Constituency < Entry self.model = "Constituencies" self.columns = "Constituency_Id,Name,ONSCode,StartDate,EndDate" - self.filter = "EndDate%20eq%20null" + self.filter = "(EndDate gt datetime'2015-05-07') or (EndDate eq null)" + self.orderby = "ONSCode" self.klass = Constituency end end diff --git a/app/lib/feed/constituency_regions.rb b/app/lib/feed/constituency_regions.rb index cb9929649..edfd66b37 100644 --- a/app/lib/feed/constituency_regions.rb +++ b/app/lib/feed/constituency_regions.rb @@ -7,7 +7,7 @@ class ConstituencyRegion < Entry self.model = "ConstituencyAreas" self.columns = "Area_Id,Constituency_Id" - self.filter = "Area/AreaType_Id%20eq%208%20and%20Constituency/EndDate%20eq%20null" + self.filter = "(Area/AreaType_Id eq 8) and ((Constituency/EndDate gt datetime'2015-05-07') or (Constituency/EndDate eq null))" self.klass = ConstituencyRegion end end diff --git a/app/lib/feed/departments.rb b/app/lib/feed/departments.rb index 97cfe652c..45e69dd08 100644 --- a/app/lib/feed/departments.rb +++ b/app/lib/feed/departments.rb @@ -11,7 +11,8 @@ class Department < Entry self.model = "Departments" self.columns = "Department_Id,Name,Acronym,Url,StartDate,EndDate" - self.filter = "EndDate%20eq%20null%20" + self.filter = "EndDate eq null" + self.orderby = "Department_Id" self.klass = Department end end diff --git a/app/lib/feed/members.rb b/app/lib/feed/members.rb index 6fe6cf67c..6f0ad7c3a 100644 --- a/app/lib/feed/members.rb +++ b/app/lib/feed/members.rb @@ -10,7 +10,8 @@ class Member < Entry self.model = "Members" self.columns = "Member_Id,NameFullTitle,Party,MembershipFrom_Id,StartDate" - self.filter = "CurrentStatusActive%20eq%20true%20and%20House_Id%20eq%201" + self.filter = "(CurrentStatusActive eq true) and (House_Id eq 1)" + self.orderby = "Member_id" self.klass = Member end end diff --git a/app/lib/feed/regions.rb b/app/lib/feed/regions.rb index f5db1222e..aec26e88b 100644 --- a/app/lib/feed/regions.rb +++ b/app/lib/feed/regions.rb @@ -8,7 +8,8 @@ class Region < Entry self.model = "Areas" self.columns = "Area_Id,Name,OnsAreaId" - self.filter = "AreaType_Id%20eq%208" + self.filter = "AreaType_Id eq 8" + self.orderby = "OnsAreaId" self.klass = Region end end diff --git a/spec/jobs/fetch_constituencies_job_spec.rb b/spec/jobs/fetch_constituencies_job_spec.rb index a8d1aeb77..062c927e0 100644 --- a/spec/jobs/fetch_constituencies_job_spec.rb +++ b/spec/jobs/fetch_constituencies_job_spec.rb @@ -2,11 +2,11 @@ RSpec.describe FetchConstituenciesJob, type: :job do let(:url) { "http://data.parliament.uk/membersdataplatform/open/OData.svc" } - let(:constituency_api) { "#{url}/Constituencies?$filter=EndDate%20eq%20null&$select=Constituency_Id,Name,ONSCode" } + let(:constituency_api) { "#{url}/Constituencies?$filter=(EndDate%20gt%20datetime'2015-05-07')%20or%20(EndDate%20eq%20null)&$orderby=ONSCode&$select=Constituency_Id,Name,ONSCode,StartDate,EndDate" } let(:stub_constituency_api) { stub_request(:get, constituency_api) } - let(:member_api) { "#{url}/Members?$filter=CurrentStatusActive%20eq%20true%20and%20House_Id%20eq%201&$select=Member_Id,NameFullTitle,Party,MembershipFrom_Id,StartDate" } + let(:member_api) { "#{url}/Members?$filter=(CurrentStatusActive%20eq%20true)%20and%20(House_Id%20eq%201)&$orderby=Member_id&$select=Member_Id,NameFullTitle,Party,MembershipFrom_Id,StartDate" } let(:stub_member_api) { stub_request(:get, member_api) } - let(:regions_api) { "#{url}/ConstituencyAreas?$filter=Area/AreaType_Id%20eq%208%20and%20Constituency/EndDate%20eq%20null&$select=Area_Id,Constituency_Id"} + let(:regions_api) { "#{url}/ConstituencyAreas?$filter=(Area/AreaType_Id%20eq%208)%20and%20((Constituency/EndDate%20gt%20datetime'2015-05-07')%20or%20(Constituency/EndDate%20eq%20null))&$orderby=Area_Id,Constituency_Id&$select=Area_Id,Constituency_Id"} let(:stub_regions_api) { stub_request(:get, regions_api) } def odata_response(status, body = nil, &block) diff --git a/spec/jobs/fetch_regions_job_spec.rb b/spec/jobs/fetch_regions_job_spec.rb index d3bb56ff0..3b4214de9 100644 --- a/spec/jobs/fetch_regions_job_spec.rb +++ b/spec/jobs/fetch_regions_job_spec.rb @@ -2,7 +2,7 @@ RSpec.describe FetchRegionsJob, type: :job do let(:url) { "http://data.parliament.uk/membersdataplatform/open/OData.svc" } - let(:regions_api) { "#{url}/Areas?$filter=AreaType_Id%20eq%208&$select=Area_Id,Name,OnsAreaId"} + let(:regions_api) { "#{url}/Areas?$filter=AreaType_Id%20eq%208&$orderby=OnsAreaId&$select=Area_Id,Name,OnsAreaId"} let(:stub_regions_api) { stub_request(:get, regions_api) } def odata_response(status, body = nil, &block) From f27f287c530896dc3231242fba6c53057c350e7b Mon Sep 17 00:00:00 2001 From: Andrew White <andrew.white@unboxed.co> Date: Mon, 3 Jun 2024 07:37:43 +0100 Subject: [PATCH 7/7] Scope constituency search to current constituencies Note that we don't have to do this for `find_by_postcode` as Parliament's API will only return us the current constituency. --- app/controllers/local_petitions_controller.rb | 4 +- app/models/constituency.rb | 4 ++ config/brakeman.ignore | 64 +++++++++---------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/app/controllers/local_petitions_controller.rb b/app/controllers/local_petitions_controller.rb index 353830c69..ecabb44a7 100644 --- a/app/controllers/local_petitions_controller.rb +++ b/app/controllers/local_petitions_controller.rb @@ -46,11 +46,11 @@ def postcode? end def find_by_postcode - @constituency = Constituency.find_by_postcode(@postcode) + @constituency = Constituency.current.find_by_postcode(@postcode) end def find_by_slug - @constituency = Constituency.find_by_slug!(params[:id]) + @constituency = Constituency.current.find_by_slug!(params[:id]) end def constituency? diff --git a/app/models/constituency.rb b/app/models/constituency.rb index 10597b995..d92ef3355 100644 --- a/app/models/constituency.rb +++ b/app/models/constituency.rb @@ -61,6 +61,10 @@ def find_by_postcode(postcode) end end + def current + where(end_date: nil) + end + def english where(arel_table[:ons_code].matches('E%')) end diff --git a/config/brakeman.ignore b/config/brakeman.ignore index adda61ffc..e5f2e9a81 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -34,22 +34,45 @@ ], "note": "" }, + { + "warning_type": "SSL Verification Bypass", + "warning_code": 71, + "fingerprint": "83faaaee2d372a0a73dc703bf46452d519d79dbf3b069a5007f71392ec7d4a3e", + "check_name": "SSLVerify", + "message": "SSL certificate verification was bypassed", + "file": "features/support/ssl_server.rb", + "line": 97, + "link": "https://brakemanscanner.org/docs/warning_types/ssl_verification_bypass/", + "code": "Net::HTTP.new(host, @port).verify_mode = OpenSSL::SSL::VERIFY_NONE", + "render_path": null, + "location": { + "type": "method", + "class": "Capybara::Server", + "method": "responsive?" + }, + "user_input": null, + "confidence": "High", + "cwe_id": [ + 295 + ], + "note": "" + }, { "warning_type": "Cross-Site Scripting", "warning_code": 4, - "fingerprint": "07b7188ce44b7041f5729077eea749b2def4b8e62736ba248267e3c96c1ca927", + "fingerprint": "859022bb61c3d1af5cdb14424490f6d3970c5b7bddd3784f62efb4f01e8fe02b", "check_name": "LinkToHref", "message": "Potentially unsafe model attribute in `link_to` href", "file": "app/views/local_petitions/all.html.erb", "line": 11, "link": "https://brakemanscanner.org/docs/warning_types/link_to_href", - "code": "link_to(Constituency.find_by_slug!(params[:id]).mp_name, Constituency.find_by_slug!(params[:id]).mp_url, :rel => \"external\")", + "code": "link_to(Constituency.current.find_by_slug!(params[:id]).mp_name, Constituency.current.find_by_slug!(params[:id]).mp_url, :rel => \"external\")", "render_path": [ { "type": "controller", "class": "LocalPetitionsController", "method": "all", - "line": 30, + "line": 32, "file": "app/controllers/local_petitions_controller.rb", "rendered": { "name": "local_petitions/all", @@ -61,7 +84,7 @@ "type": "template", "template": "local_petitions/all" }, - "user_input": "Constituency.find_by_slug!(params[:id]).mp_url", + "user_input": "Constituency.current.find_by_slug!(params[:id]).mp_url", "confidence": "Weak", "cwe_id": [ 79 @@ -71,19 +94,19 @@ { "warning_type": "Cross-Site Scripting", "warning_code": 4, - "fingerprint": "22e002a1359fd28418d81e2cadeb49195a5597840a43d97787ac79a868acb51f", + "fingerprint": "b44e200c1415ee4d50599d5a9854799a8de42354f84c7530d5c382a35fe2547e", "check_name": "LinkToHref", "message": "Potentially unsafe model attribute in `link_to` href", "file": "app/views/local_petitions/show.html.erb", "line": 11, "link": "https://brakemanscanner.org/docs/warning_types/link_to_href", - "code": "link_to(Constituency.find_by_slug!(params[:id]).mp_name, Constituency.find_by_slug!(params[:id]).mp_url, :rel => \"external\")", + "code": "link_to(Constituency.current.find_by_slug!(params[:id]).mp_name, Constituency.current.find_by_slug!(params[:id]).mp_url, :rel => \"external\")", "render_path": [ { "type": "controller", "class": "LocalPetitionsController", "method": "show", - "line": 22, + "line": 24, "file": "app/controllers/local_petitions_controller.rb", "rendered": { "name": "local_petitions/show", @@ -95,36 +118,13 @@ "type": "template", "template": "local_petitions/show" }, - "user_input": "Constituency.find_by_slug!(params[:id]).mp_url", + "user_input": "Constituency.current.find_by_slug!(params[:id]).mp_url", "confidence": "Weak", "cwe_id": [ 79 ], "note": "" }, - { - "warning_type": "SSL Verification Bypass", - "warning_code": 71, - "fingerprint": "83faaaee2d372a0a73dc703bf46452d519d79dbf3b069a5007f71392ec7d4a3e", - "check_name": "SSLVerify", - "message": "SSL certificate verification was bypassed", - "file": "features/support/ssl_server.rb", - "line": 97, - "link": "https://brakemanscanner.org/docs/warning_types/ssl_verification_bypass/", - "code": "Net::HTTP.new(host, @port).verify_mode = OpenSSL::SSL::VERIFY_NONE", - "render_path": null, - "location": { - "type": "method", - "class": "Capybara::Server", - "method": "responsive?" - }, - "user_input": null, - "confidence": "High", - "cwe_id": [ - 295 - ], - "note": "" - }, { "warning_type": "Cross-Site Scripting", "warning_code": 114, @@ -164,6 +164,6 @@ "note": "" } ], - "updated": "2024-05-10 12:37:54 +0000", + "updated": "2024-05-31 17:06:26 +0000", "brakeman_version": "6.1.2" }