anonymize_script_template_v2.do

********************************************************************************
** 	TITLE: 		anonymize.do
**
**	PURPOSE: 	This do file anonymizes datasets based on users instruction
**
**	NOTES:		This scrpit was automatically generated by IPA's PII detector app. For details, check https://github.com/PovertyAction/PII_detection
**
**	AUTHOR:      PII Detector app
**
**	CREATED: 	[date]
********************************************************************************
*Table of Contents:
*1. Set Directories
*2. Strip PII
*3. Anonymize IDs


clear all
cap log close

version 14.2
set more off
set varabbrev off


***************************************
* 0. Define HMAC-SHA1 Program
***************************************
/*
	We use HMAC-SHA1 to hash any PII variables. This is to ensure that
	hashing can be consistently done between computers and statistical software.

	We use William Matsuoka's implementation of HMAC-SHA1 from:
		http://www.wmatsuoka.com/stata/hmac-sha1-in-stata
*/

mata:

	string scalar hmac_sha1(string scalar key, string scalar message)
	{
		blocksize = 64
		if (strlen(key) > blocksize) {
			key = sha1(key)
			key = base16to64(key)
		}
		if (strlen(key) < blocksize) {
			key = key + char(0) * (blocksize - strlen(key))
		}

		key = inbase(2, ascii(key))
		key = invtokens("0" :* (8:-strlen(key)) :+ key, "")


		okp = o_key_pad(blocksize, key)
		okp = base2ascii(okp)

		ikp = i_key_pad(blocksize, key)
		ikp = base2ascii(ikp)

		h1 = base16to64( sha1(ikp + message) )
		hmac = sha1(okp + h1)
		return(hmac)
	}
	string matrix o_key_pad(real scalar blocksize, string scalar key)
	{
		bit = J(1, strlen(key), "")
		opad = inbase(2, ascii(char(92)* blocksize))
		opad = invtokens("0" :* (8:-strlen(opad)) :+ opad, "")

		bit = bitxor(key, opad)

		return (bit)
	}
	string matrix i_key_pad(real scalar blocksize, string scalar key)
	{
		bit = J(1, strlen(key), "")
		ipad = inbase(2, ascii(char(54)* blocksize))
		ipad = invtokens("0" :* (8:-strlen(ipad)) :+ ipad, "")

		bit = bitxor(key, ipad)

		return (bit)
	}
	string scalar base2ascii(string scalar x)
	{
		y = J(1, strlen(x)/8, "")
		for(i=1; i<=strlen(x)/8; i++) {
			y[i] = char(frombase(2, substr(x, (i-1)*8 + 1, 8)))
		}
		return(invtokens(y, ""))
	}
	string scalar sha1toascii(string scalar x)
	{
		y = J(1, strlen(x)/2, "")
		for(i=1; i<=strlen(x)/2; i++) {
			y[i] = char(frombase(16, substr(x, (i-1)*2 + 1, 2)))
		}
		return (invtokens(y, ""))
	}

	// SHA1
	string scalar sha1(string scalar message)
	{
		// Find Blocks and Padding
		m_a     = strlen(message) * 8
		message = message + char(128)
		m_l     = strlen(message) * 8
		m_l_b   = padbit(inbase(2, m_a), 64)
		blocks  = trunc((m_l + 64)/512) + 1
		total   = blocks * 512

		bit = inbase(2, ascii(message))
		bit = "0" :* (8 :- strlen(bit)) :+ bit
		bit = invtokens(bit, "")

		message = bit + "0"*(total - m_l - 64) + m_l_b

		// Initialize
		h0 = "01100111010001010010001100000001"
		h1 = "11101111110011011010101110001001"
		h2 = "10011000101110101101110011111110"
		h3 = "00010000001100100101010001110110"
		h4 = "11000011110100101110000111110000"

		for (i=1; i<=blocks; i++) {
			x = substr(message, (i-1)*512 + 1, 512)
			sha1_proc(x, h0, h1, h2, h3, h4)
		}

		hh = pb6to2(h0) + pb6to2(h1) + pb6to2(h2) + pb6to2(h3) + pb6to2(h4)
		return (hh)
	}
	void sha1_proc(string scalar bit, string scalar h0, string scalar h1,
		string scalar h2, string scalar h3, string scalar h4)
	{
		// break word into 32 bit chunks
		w = J(1, 80, "")
		for (i = 1; i<= 16; i++) {
			w[i] = substr(bit, (i-1)*32 + 1, 32)
		}

		// extend rest of words
		for (i=17; i<=80; i++) {
			w[i] = bitxor( bitxor( bitxor(w[i-3] , w[i-8]) , w[i-14]) , w[i-16] )
			w[i] = leftrotate(w[i],1)
		}

		// initalize hash
		a = h0
		b = h1
		c = h2
		d = h3
		e = h4

		for (i=1; i<=80; i++) {
			if (i <= 20) {
				f = bitor(bitand(b, c) , bitand(bitnot(b), d))
				k = "01011010100000100111100110011001"
			}
			else if (i <= 40) {
				f = bitxor(bitxor(b, c), d)
				k = "01101110110110011110101110100001"
			}
			else if (i <= 60) {
				f = bitor(bitor(bitand(b, c) , bitand(b,d)) , bitand(c, d))
				k = "10001111000110111011110011011100"
			}
			else {
				f = bitxor(bitxor(b, c), d)
				k = "11001010011000101100000111010110"
			}

			temp = inbase(2,
						frombase(2, leftrotate(a, 5)) +
						frombase(2, f) + frombase(2, e) +
						frombase(2, k) + frombase(2, w[i]) )
			e = overflow(d)
			d = overflow(c)
			c = overflow(leftrotate(b, 30))
			b = overflow(a)
			a = overflow(temp)
		}

		h0 = overflow(inbase(2, frombase(2, h0) + frombase(2, a)))
		h1 = overflow(inbase(2, frombase(2, h1) + frombase(2, b)))
		h2 = overflow(inbase(2, frombase(2, h2) + frombase(2, c)))
		h3 = overflow(inbase(2, frombase(2, h3) + frombase(2, d)))
		h4 = overflow(inbase(2, frombase(2, h4) + frombase(2, e)))

	}
	string scalar pb6to2(string scalar x)
	{
		x = inbase(16, frombase(2, x))
		return(padbit(x, 8))
	}
	string scalar bitxor(string scalar bit1, string scalar bit2)
	{
		if (strlen(bit1) != strlen(bit2)) {
			errprintf("Your two strings must be of the same length\n")
			exit(198)
		}
		b1 = b2 = b3 = J(1, strlen(bit1), "")
		for (i=1; i<=strlen(bit1); i++) {
			b1[i] = substr(bit1, i, 1)
			b2[i] = substr(bit2, i, 1)
			b3[i] = strofreal(b1[i] != b2[i])
		}
		t = invtokens(b3, "")
		return(t)
	}
	string scalar bitand(string scalar bit1, string scalar bit2)
	{
		if (strlen(bit1) != strlen(bit2)) {
			errprintf("Your two strings must be of the same length\n")
			exit(198)
		}
		b1 = b2 = b3 = J(1, strlen(bit1), "")
		for (i=1; i<=strlen(bit1); i++) {
			b1[i] = substr(bit1, i, 1)
			b2[i] = substr(bit2, i, 1)
			b3[i] = strofreal((b1[i]=="1" & b2[i]=="1"))
		}
		t = invtokens(b3, "")
		return(t)
	}
	string scalar bitor(string scalar bit1, string scalar bit2)
	{
		if (strlen(bit1) != strlen(bit2)) {
			errprintf("Your two strings must be of the same length\n")
			exit(198)
		}
		b1 = b2 = b3 = J(1, strlen(bit1), "")
		for (i=1; i<=strlen(bit1); i++) {
			b1[i] = substr(bit1, i, 1)
			b2[i] = substr(bit2, i, 1)
			b3[i] = strofreal((b1[i]=="1" | b2[i]=="1"))
		}
		t = invtokens(b3, "")
		return(t)
	}
	string scalar bitnot(string scalar bit1)
	{
		b1 = J(1, strlen(bit1), "")
		for (i=1; i<=strlen(bit1); i++) {
			b1[i] = strofreal((substr(bit1, i, 1) == "0"))
		}
		return (invtokens(b1, ""))
	}
	string scalar leftrotate(string scalar x, real scalar i)
	{
		l_x = strlen(x)
		s_r = mod(i, l_x)

		if (s_r != 0) {
			a1 = substr(x, 1, s_r)
			a2 = substr(x, s_r + 1, .)
			re = a2 + a1
		}
		else {
			re = x
		}
		return (re)
	}
	string scalar leftshift(string scalar x, real scalar i)
	{
		l_x = strlen(x)
		if (i > 0) {
			a   = substr(x, i + 1, .)
			l_a = strlen(a)
			re  = a + "0" * (l_x-l_a)
		}
		else {
			re = x
		}
		return (re)
	}
	string scalar padbit(string scalar x, real scalar i, |real scalar d)
	{
		l_x = strlen(x)
		if (i <= l_x) return(x)

		if (args() == 3) {
			x = x + "0"*(i-l_x)
			return(x)
		}
		else {
			x = "0"*(i-l_x) + x
			return(x)
		}
	}
	string scalar overflow(string scalar x)
	{
		y = mod(frombase(2, x), 2^32)
		y = padbit(inbase(2, y), 32)
		return(y)
	}
	string scalar base16to64(string scalar x)
	{
		y = J(1, strlen(x)/2, .)
		for(i=1; i<=strlen(x)/2; i++) {
			y[i] = frombase(16, substr(x, (i-1)*2 + 1, 2))
		}
		return(char(y))
	}

end


***************************************
* 1. Set directories
***************************************
*A. Set locations
*B. Set up varlists for modification
*C. Use file


**A. Set locations
	local infile 	"[input_file_path]"
	local outfile 	"[output_file_path]"


**B. Set varlists for modification
    loc dropvars 	[list_variables_to_drop_space_delimited]
    loc valvars 	[list_variables_to_remove_value_labelling_space_delimited]
    loc hashvars 	[list_variables_to_hash_space_delimited]


**C. Use files
	use "`infile'", clear



***************************************
* 2. Strip PII
***************************************
*A. Drops the variables
*B. Removes value labeling
*C. Converts to an encoded value and drops the value labeling
*D. Hash an ID

/*
	This section remove information, either variables or labels, to anonymize
	the data based on selections in the PII detector .exe.

	It does so in three ways:

		A. Removes explicit PII variables
		B. Removes labels of encoded variables that contain PII
		C. Encodes string values and drops the label
		D. Hash ID values to ensure public data cannot merge on to internal
		   data easily.

	Before you produce the fully anonymized dataset ensure that you confirm PII
	is removed by looking at remaining string variables and checking the full
	 variable list as well as checking notes are empty:

		ds, v(32)
		ds, has(type string) v(32)
		notes
*/



**A. Drop variables
	drop `dropvars'


**B. Remove value labeling
	foreach var of local valvars {

		* Get value label name
		loc vallab : value label `var'

		*Remove labeling
		if "`vallab'" != "" {

			*Remove value label from variable
			label values `var' 													// Stata requires value labels to be stripped before dropping
			label drop `vallab' 												// Remove value labels from memory
		}
		// end if "`vallab'" existis

	}
	// end foreach var of local valvars

**C.

**D. Hash an ID
	/*
		We hash the vairable using the hashfunction. It requires inputs as 
		strings, so variables are treated differently based on storage format:

			(1) String 			- No change
			(2) Encoded numeric - Converted to string and reencoded to 1-N
			(3) Numeric 		- Converted to string with display format
	*/
	foreach var of local hashvars {

		di "Hashing variable `var'"

		*Save label
		loc varlab : variable label `var'
		
		* Convert format to string based on value label
		cap confirm string variable `var' 											// Ensure only string variables to hash
		if _rc & "`: value label `var''" != "" { 									// Decode labeled variables

			* Create hashed variable
			tempvar `var'_dec
			decode `var', gen(``var'_dec') 											// Decode to string while retaining the value label

			*Replace variable with hashed output
			drop `var'
			qui gen `var' = ``var'_dec'
			label var `var' "`varlab'"
		}
		else if _rc & "`: value label `var''" == "" { 								// Encode labeled variables
			tostring `var', replace usedisplayformat 
		}

		*Create tempvar
		qui levelsof `var', loc(hash`var')
		loc `var'_n = `r(r)'

		forvalues i = 1(1)``var'_n' {

			*Save local hashvar
			loc loc`i' "`: word `i' of `hash`var'''"
			mata: st_local("v`i'", hmac_sha1("[SECRET KEY]", "`loc`i''"))

			*Check all values exist
			qui count if `var' == "`: word `i' of `hash`var'''"
			assert `r(N)' != 0

			* Replace hashed value into tempvar
			qui replace `var' = "`v`i''" if `var' == "`: word `i' of `hash`var'''"
		}
		// end forvalues i = 1(1)levels

		* Create hashed variable
		tempvar `var'_enc
		encode `var', gen(``var'_enc') 											// Encode to numeric in alphabetical order of hash
		lab values ``var'_enc' 													// String hash

		*Replace variable with hashed output
		drop `var'
		qui gen `var' = ``var'_enc'
		label var `var' "`varlab'"
	}
	// end foreach var of local hashvars



***************************************
* 3. Save file
***************************************
	*Save file
	cap drop __* 																// Ensure no tempvar artifiacts are saved
	save "`outfile'", replace



**EOF**