Merge branch 'release/1.0.0-alpha.2'

.gitignore

@@ -19,7 +19,6 @@ buildNumber.properties
 ### Eclipse template
 
 .metadata
-bin/
 tmp/
 *.tmp
 *.bak
@@ -87,13 +86,24 @@ consonance/key.pem
 consonance/bag_params.json
 consonance/config
 consonance/youxia_config
+consonance/web.yml
+consonance/boto.config
+
+# redwood files
+action/aws.config
+action/.env
+action_launcher_config/action.config
 
 redwood/aws.config
 redwood/.env
 redwood_launcher_config/redwood.config
+redwood/backups
 
 boardwalk/.env
 boardwalk/aws.config
+boardwalk/dcc-dashboard-service/
+boardwalk/dcc-dashboard/
+boardwalk/dcc-metadata-indexer/
 boardwalk_launcher_config/boardwalk.config
 
 aws.config
@@ -105,10 +115,18 @@ install_wget_curl.log
 redwood/manifest.tsv
 youxia_config
 
+#Reference folder
+reference/samples/
+reference/outputs/
+
 # vagrant
 .vagrant
 ubuntu-xenial-16.04-cloudimg-console.log
 
+# certificates
+common/certs
+
 # log files
 install_compose.log
 install_docker.log
+install_pip_aws.log

action/action_compose.yml

@@ -0,0 +1,30 @@
+version: '2'
+
+services:
+  dcc-action:
+    image: quay.io/ucsc_cgl/action-service:1.0.4
+    container_name: "action-service"
+    environment:
+      STORAGE_ACCESS_TOKEN: "${storage_access_token}"
+      STORAGE_SERVER: "${storage_server}"
+      ELASTIC_SEARCH_SERVER: "${elastic_search_server}"
+      ELASTIC_SEARCH_PORT: "${elastic_search_port}"
+      TOUCH_FILE_DIRECTORY: "${touch_file_directory}"
+      CONSONANCE_WEB_SERVICE_URL: "${consonance_web_service_url}"
+      CONSONANCE_ACCESS_TOKEN: "${consonance_access_token}"
+      AWS_SECRET_ACCESS_KEY: "${aws_secret_access_key}"
+      AWS_ACCESS_KEY_ID: "${aws_access_key_id}"
+      AWS_REGION: "${aws_region}"
+      DOCKSTORE_SERVER_URL: "${dockstore_server_url}"
+      DOCKSTORE_TOKEN: "${dockstore_token}"
+    volumes:
+      - ~/dcc-action-service/logs:/home/ubuntu/logs
+    restart: always
+    ports:
+      - "8082"
+    networks:
+      - esnet
+
+networks:
+  esnet:
+    external: true

action/conf/action.config.template

@@ -0,0 +1,13 @@
+dockstore_server_url={{ DOCKSTORE_SERVER_URL }}
+dockstore_token={{ DOCKSTORE_TOKEN }}
+consonance_web_service_url={{ CONSONANCE_WEB_SERVICE_URL }}
+consonance_access_token={{ CONSONANCE_ACCESS_TOKEN }}
+storage_server={{ STORAGE_SERVER }}
+storage_access_token={{ STORAGE_ACCESS_TOKEN }}
+elastic_search_server={{ ELASTIC_SEARCH_SERVER }}
+elastic_search_port={{ ELASTIC_SEARCH_PORT }}
+touch_file_directory={{ TOUCH_FILE_DIRECTORY }}
+aws_secret_access_key={{ AWS_SECRET_ACCESS_KEY }}
+aws_access_key_id={{ AWS_ACCESS_KEY_ID }}
+aws_region={{ AWS_REGION }}
+aws_profile={{ AWS_PROFILE }}

boardwalk/README.md

@@ -9,27 +9,77 @@ Boardwalk is the core's file browser to display the metadata of files stored in
 The metadata-indexer pulls the metadata information from redwood and creates a donor oriented index. It then uses this index to create a file oriented index. Both of these indexes get loaded into elasticsearch. The dashboard-service then reads from elasticsearch and returns queries that can be used to populate the Boardwalk file browser. 
 
 ## Deployment
-Deployment is done through the `install_bootstrap` script on the root directory. Currently, the development version is just available; put `dev` when prompted for which mode to run on.
+Deployment is done through the `install_bootstrap` script on the root directory. It will ask you a series of question to collect the information required to install boardwalk and its components.
 
-Before proceeding, you must ensure that you have a Google OAuth2 app. Follow the instructions on [here](http://bitwiser.in/2015/09/09/add-google-login-in-flask.html#creating-a-google-project) under "Creating A Google Project"
+### Create a Google Oauth2 app
 
-You need to collect the following information as well before launching the installation:
+Before proceeding, you must ensure that you have a Google OAuth2 app. Follow the instructions on [here](http://bitwiser.in/2015/09/09/add-google-login-in-flask.html#creating-a-google-project) under "Creating A Google Project". 
 
-* Google Client ID; Go to `https://console.developers.google.com/` and look for your Oauth2 app. 
-* Google Client Secret; Go to `https://console.developers.google.com/` and look for your Oauth2 app. 
+If you don't want to enable Login and token retrieval through the portal, you can just put some random string for the Google Client ID and Google Client Secret. 
+
+Here is a summary of what you need to do in order to create a Google OAuth2 app:
+
+* Go to [Google's Developer Console](https://console.developers.google.com/).
+* On the upper left side of the screen, click on the drop down button.
+* Create a project by clicking on the plus sign on the pop-up window.
+* On the next pop up window, add a name for your project. 
+* Once you create it, click on the "Credentials" section on the left hand side.
+* Click on the "OAuth Consent Screen". Fill out a product name and choose an email for the Google Application. Fill the rest of the entries as you see fit for your purposes, or leave them blank, as they are optional. Save it.
+* Go to the "Credentials" tab. Click on the "Create Credentials" drop down menu and choose "OAuth Client ID".
+* Choose "Web Application" from the menu. Assign it a name. 
+* Under "Authorized JavaScript origins", enter `http://<YOUR_SITE>`. Press Enter. Add a second entry, same as the first one, but use *https* instead of *http*
+* Under "Authorized redirect URIs", enter `http://<YOUR_SITE>/gCallback`. Press Enter. Add a second entry, same as the first one, but use *https* instead of *http*
+* Click "Create". A pop up window will appear with your Google Client ID and Google Client Secret. Save these. If you lose them, you can always go back to the Google Console, and click on your project; the information will be there. Keep these values stored in a safe location. Treat them as you would treat a credit card number. 
+
+**Please note:** at this point, the dashboard only accepts login from emails with a 'ucsc.edu' domain. In the future, it will support different email domains. 
+
+### Development Mode
+
+The boardwalk installer has the option to install boardwalk and its components in either devevelopment or production mode (dev/prod). If you are installing in production, skip this section and head to **Installation Questions**. Otherwise, keep reading.
+
+To make deployment during development faster, dev mode will assume you have cloned *dcc-dashboard, dcc-dashboard-service, and dcc-metadata-indexer* under *dcc-ops/boardwalk* If you are running dcc-ops in dev mode and you haven't already, clone the repos by running the following from within `dcc-ops/boardwalk`:
+
+```
+git clone https://github.com/BD2KGenomics/dcc-dashboard-service.git
+git clone https://github.com/BD2KGenomics/dcc-metadata-indexer.git
+git clone https://github.com/BD2KGenomics/dcc-dashboard.git
+```
+Make sure you check the branches you will be doing development on within each of the cloned repos.
+
+Once you run the installer, docker-compose will use dev.yml to set up boardwalk and its components. It will create the Docker images using the Dockerfiles located inside `dcc-dashboard, dcc-dashboard-service, and dcc-metadata-indexer/v2`. 
+
+In addition, installing boardwalk in dev mode will also install kibana under `myexample.com/kibana/` to aid in debugging all things related to elasticsearch, as well as to help in making new queries and aggregations that may be necessary. 
 
 ### Installation Questions
+* Choose a mode you want to run the installer (prod/dev). 
 * On question `What is your Google Client ID?`, put your Google Client ID. See [here](http://bitwiser.in/2015/09/09/add-google-login-in-flask.html#creating-a-google-project)
 * On question `What is your Google Client Secret?`, put your Google Client Secret. See [here](http://bitwiser.in/2015/09/09/add-google-login-in-flask.html#creating-a-google-project)
 * On question `What is your DCC Dashboard Host?`, put the domain name resolving to your Virtual Machine (e.g. `example.com`)
 * On question `What is the user and group that should own the files from the metadata-indexer?`, type the `USER:GROUP` pair you want the files downloaded by the indexer to be owned by. The question will show the current `USER:GROUP` pair for the current home directory. Highly recommended to type the same value in there (e.g. `1000:1000`)
 * On question `How should the database for billing should be called?`, type the name to be assigned to the billing database.
 * On question `What should the username be for the billing database?`, type the username for the billing database.
 * On question `What should the username password be for the billing database?`, type some password for the billing database. 
-* On question `What is the AWS profile?`, type some random string (DEV)
-* On question `What is the AWS Access key ID?`, type some random string (DEV)
-* On question `What is the AWS secret access key?`, type some random string (DEV)
-* On question `What is the Luigi Server?`, type some random string (DEV)
+* On question `What is the AWS profile?`, type some random string (DEV, PROD)
+* On question `What is the AWS Access key ID?`, type some random string (DEV, PROD)
+* On question `What is the AWS secret access key?`, type some random string (DEV, PROD)
+* On question `What is the Consonance Address?`, type some random string (DEV, PROD)
+* On question `What is the Consonance Token`, type some random string (DEV, PROD)
+* On question `What is the Luigi Server?`, type some random string (DEV, PROD)
 * On question `What is the Postgres Database name for the action service?`, type the name to be assigned to the action service database.
 * On question `What is the Postgres Database user for the action service?`, type the username to be assigned to the the action service database.
 * On question `What is the Postgres Database password for the action service?`, type the password to be assigned to the action service database. 
+
+### Checking Docker Containers
+
+You can check which docker containers are running by doing `sudo docker ps`. If the installation was successful, you should see the following containers running:
+```
+boardwalk_dcc-metadata-indexer_1
+boardwalk_dcc-dashboard-service_1
+boardwalk_dcc-dashboard_1
+boardwalk_nginx_1
+boardwalk_boardwalk_1
+elasticsearch1
+boardwalk_elasticsearch2_1
+boardwalk_db_1
+boardwalk-billing
+```

boardwalk/conf/boardwalk.config.template

@@ -16,11 +16,18 @@ billing_user={{BILLING_USER}}
 billing_password={{BILLING_PASSWORD}}
 user_group={{ USER_GROUP }}
 es_service=elasticsearch1
+consonance_address={{ CONSONANCE_ADDRESS }}
+consonance_token={{ CONSONANCE_TOKEN  }}
 aws_profile={{ AWS_PROFILE }}
 aws_access_key_id={{ AWS_ACCESS_KEY_ID }}
 aws_secret_access_key={{ AWS_SECRET_ACCESS_KEY }}
 luigi_server={{ LUIGI_SERVER }}
+luigi_port={{ LUIGI_PORT }}
 postgres_db={{ POSTGRES_DB }}
 postgres_user={{ POSTGRES_USER }}
 postgres_password={{ POSTGRES_PASSWORD }}
-
+login_user={{ LOGIN_POSTGRES_USER }}
+login_db={{ LOGIN_POSTGRES_DB }}
+login_password={{ LOGIN_POSTGRES_PASSWORD }}
+secret_key={{ SECRET_KEY }}
+core_client_version={{ DCC_CORE_CLIENT_VERSION }}