diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..402505fc030 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true diff --git a/.gitignore b/.gitignore index dfc7f5a9238..3cc4a24cd53 100755 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /.vagrant -/.idea \ No newline at end of file +/.idea +/log/*.log +/scripts/.current_nesting_level +/scripts/.current_log_path +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9a0045565..79aa3334cf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,87 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). * [\[Unreleased\]](#unreleased) + * [\[v2.2.0\] - 2017-02-25](#v220---2017-02-25) + * [\[v2.1.0\] - 2016-06-22](#v210---2016-06-22) * [\[v2.0.0\] - 2016-02-05](#v200---2016-02-05) * [\[v1.0.0\] - 2016-01-11](#v100---2016-01-11) ## [Unreleased] -Nothing yet +### Changed + + - Verified compatibility with Vagrant 2.x and loosened Vagrant version requirement + - Config option 'php_version' should now be used for PHP version selection instead of 'use_php7' + - Upgraded Varnish to v4.1 + - Improved Magento re-installation speed by avoiding unnecessary cache cleaning + - Custom code sniffer rules replaced with PSR2 + - MessageQueue module will now be installed with CE if using Magento v2.3+ + +### Added + + - PHP 7.1 and PHP 7.2 support + - Support for new location of `generated` directory in Magento 2.2.0 + - Basic data generation can be turned off. Added configurable product and customer address generation + - Documentation in config.yaml.dist for choosing specific composer package versions + +### Fixed + + - Magento 2.2.0 EE installation from composer + +## [v2.2.0] - 2017-02-25 + +### Changed + + - Magento EE and sample data directories are added to 'Exclude list' in PhpStorm + - Redis is used as default cache backend instead of filesystem + - After EE is linked, EE versions of composer.lock and composer.json are replaced back with CE versions (thus are not marked by git as modified) + - Improved CLI scripts output formatting + +### Fixed + + - Issue with some files marked as unversioned in git after EE linking + - It is now possible to use spaces in path to vagrant project on Windows. On OSX and Linux hosts it works in non-NFS mode, in NFS mode it does not work because of [bug in vagrant](mitchellh/vagrant#7540) + - Issue with MySQL being down after VM power off + +### Added + + - Added ability to configure number of CPUs for virtual machine via [etc/config.yaml](etc/config.yaml.dist) + - Added generation of basic sample data for testing purposes + - Ability to upgrade Magento using `m-switch-to-ce` and `m-switch-to-ee` (when `-u` flag is specified) + - Redis support for Magento caching + - Access to Magento developer mode and storefront/admin UI debugging features via [etc/config.yaml](etc/config.yaml.dist) + - Composer-based installation support + - Magento cache warming up after re-install and clearing cache (when `-w` flag is specified) + - Tests configuration files are generated during project initialization + - Sample data support + - ElasticSearch support + - NodeJS, NPM, Gulp and Grunt are installed as part of the provisioning process + +## [v2.1.0] - 2016-06-22 + +### Changed + + - Removed requirement for public github token due to Composer limitations (issue is fixed on Composer side) + - Changed requirement for minimum box version from 1.0 to 1.1 + - Upgraded PHP 5.5.9 to PHP 5.6 + - When [init_project.sh](init_project.sh) is executed, EE will be installed by default, if EE repository is specified in [etc/config.yaml](etc/config.yaml.dist). Not supported on Windows hosts + +### Fixed + + - Permissions during Magento installation on Windows hosts + - Issue with Magento compiler on Windows hosts + - "stdin: is not a tty" warning + +### Added + + - Setup and upgrade cron scripts to crontab + - Logging of all emails in HTML format to `vagrant-magento/log/email` + - Host wrapper script for bin/magento command on guest + - Ability to modify guest config files (PHP, Apache etc) directly from host IDE + - Ability to choose if PhpStorm configs should be removed during project reinitialization + - Ability to switch PHP version without destroying the project ("vagrant reload" is required) + - Ability to do force switch to CE or EE (even if already switched to target edition) + - Ability to use Varnish full page caching automatically. (Using "vagrant reload" or m-varnish script) ## [v2.0.0] - 2016-02-05 @@ -17,7 +92,7 @@ Nothing yet - Moved provisioning scripts - Magento project directory moved to vagrant project root. Current structure is as follows: `vagrant-magento/magento2ce/magento2ee` - - PHP 7.0 is installed by default instead of PHP 5.5.9 (can be configured in [etc/config.yaml.dist](etc/config.yaml.dist)) + - PHP 7.0 is installed by default instead of PHP 5.5.9 (can be configured in [etc/config.yaml](etc/config.yaml.dist)) - Renamed configuration folder from `local.config` to `etc` - Set minimum Vagrant version as 1.8 - Improved deployment speed in case of disabled NFS for folders sync @@ -30,27 +105,27 @@ Nothing yet - Added [project initialization script](init_project.sh) and host scripts for routine flows (compatible with OSX, *nix and Windows) - Implemented static value of forwarded SSH port to prevent necessity to reconfigure software accessing guest via SSH - Implemented collision prevention for IP address and host name (in case when several machines are created at once) - - Added configuration file [etc/config.yaml.dist](etc/config.yaml.dist) - - Added PHP 7.0 support - - Added PHP Storm configuration during project initialization (particularly automatic deployment settings) - - Added automatic vagrant plugins installation + - Configuration file [etc/config.yaml](etc/config.yaml.dist) + - PHP 7.0 support + - PHP Storm configuration during project initialization (particularly automatic deployment settings) + - Automatic vagrant plugins installation ## [v1.0.0] - 2016-01-11 ### Added - - Integrated vagrant host manager plugin to allow automatic /etc/hosts update - - Added support of EE linked to CE using symlinks on *nix hosts - - Added ${MAGENTO_ROOT} environment variable, which stores installation path on the guest - - Added support of Rabbit MQ - - Added possibility to specify tokens for repo.magento.com composer repository + - Integrated vagrant host manager plugin to allow automatic /etc/hosts update + - Support of EE linked to CE using symlinks on *nix hosts + - ${MAGENTO_ROOT} environment variable, which stores installation path on the guest + - Support of Rabbit MQ + - Possibility to specify tokens for repo.magento.com composer repository - git is now installed on guest machine - - Removed 'magento' MySQL user, password of 'root' user removed - - Database for integration tests are created by default - - Added script for clearing Magento cache from host command line + - Removed 'magento' MySQL user, password of 'root' user removed + - Database for integration tests are created by default + - Added script for clearing Magento cache from host command line - Configured XDebug to allow remote debugging - - Fixed max_nesting_level issue with XDebug enabled + - Fixed max_nesting_level issue with XDebug enabled - Apache is run by 'vagrant' user - - Enabled Magento cron jobs + - Enabled Magento cron jobs - Enabled XDebug by default - Created vagrant configuration for Magneto 2 CE developer's environment installation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 832c5682a44..b7de76d26fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1 @@ -Everyone is welcome to contribute improvements and fixes to this project. There are no specific requirements, just create a pull request with proposed modifications. \ No newline at end of file +Everyone is welcome to contribute improvements and fixes to this project. There are no specific requirements, just create a pull request with proposed modifications and try to run [tests](https://github.com/paliarush/magento2-vagrant-for-developers-tests) on your local environment. \ No newline at end of file diff --git a/README.md b/README.md index e0eb28b1937..31ae3497aca 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Vagrant project for Magento 2 developers (optimized for Mac, Windows and \*nix hosts) +[![Tests passing on OSX](docs/images/tests_badge.png)](https://github.com/paliarush/magento2-vagrant-for-developers-tests) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) [![Semver](http://img.shields.io/SemVer/2.0.0.png?color=blue)](http://semver.org/spec/v2.0.0.html) [![Latest GitHub release](docs/images/release_badge.png)](https://github.com/paliarush/magento2-vagrant-for-developers/releases/latest) @@ -10,86 +11,116 @@ * [Installation steps](#installation-steps) * [Default credentials and settings](#default-credentials-and-settings) * [Getting updates and fixes](#getting-updates-and-fixes) - * [GitHub limitations](#github-limitations) * [Day-to-day development scenarios](#day-to-day-development-scenarios) * [Reinstall Magento](#reinstall-magento) - * [Clear magento cache](#clear-magento-cache) - * [Update composer dependencies](#update-composer-dependencies) + * [Clear Magento cache](#clear-magento-cache) * [Switch between CE and EE](#switch-between-ce-and-ee) + * [Sample data installation](#sample-data-installation) + * [Basic data generation](#basic-data-generation) + * [Use Magento CLI (bin/magento)](#use-magento-cli-binmagento) * [Debugging with XDebug](#debugging-with-xdebug) + * [Connecting to MySQL DB](#connecting-to-mysql-db) + * [View emails sent by Magento](#view-emails-sent-by-magento) + * [Accessing PHP and other config files](#accessing-php-and-other-config-files) + * [Upgrading Magento](#upgrading-magento) * [Multiple Magento instances](#multiple-magento-instances) + * [Update Composer dependencies](#update-composer-dependencies) + * [Running Magento tests](#running-magento-tests) + * [Environment configuration](#environment-configuration) + * [Switch between PHP versions](#switch-between-php-versions) + * [Activating Varnish](#activating-varnish) + * [Activating ElasticSearch](#activating-elasticsearch) + * [Redis for caching](#redis-for-caching) + * [Reset environment](#reset-environment) + * [Switch NodeJS Versions](#switch-nodejs-versions) + * [FAQ](#faq) ## What You get -It is expected that Magento 2 project source code will be located and managed on the host. This is necessary to allow quick indexing of project files by IDE. All other infrastructure is deployed on the guest machine. +It's expected that the Magento 2 project source code will be located and managed on the host to allow quick indexing of project files by IDE. All other infrastructure is deployed on the guest machine. -Current Vagrant configuration aims to solve performance issues of Magento installed on Virtual Box **for development**. Custom solution is implemented for Windows hosts. See [explanation of the proposed solution](docs/performance-issue-on-windows-hosts.md). +Current Vagrant configuration aims to solve performance issues of Magento installed on Virtual Box **for development**. A custom solution is implemented for Windows hosts. See [explanation of the proposed solution](docs/performance-issue-on-windows-hosts.md). -Environment for Magento EE development is configured as well. +The environment for Magento EE development is also configured. It is easy to [install multiple Magento instances](#multiple-magento-instances) based on different codebases simultaneously. -[Project initialization script](init_project.sh) configures complete development environment: +The [project initialization script](init_project.sh) configures a complete development environment: 1. Adds some missing software on the host - 1. Configures all software necessary for Magento 2 using [custom Ubuntu vagrant box](https://atlas.hashicorp.com/paliarush/boxes/magento2.ubuntu) (Apache 2.4, PHP 7.0 (or 5.5.9), MySQL 5.6, Git, Composer, XDebug, Rabbit MQ) - 1. Installs Magento 2 + 1. Configures all software necessary for Magento 2 using a [custom Ubuntu vagrant box](https://atlas.hashicorp.com/paliarush/boxes/magento2.ubuntu) (Apache 2.4, PHP 7.0 or 5.6, MySQL 5.6, Git, Composer, XDebug, Rabbit MQ, Varnish) + 1. Installs Magento 2 from Git repositories or Composer packages (can be configured via `checkout_source_from` option in [etc/config.yaml](etc/config.yaml.dist)) 1. Configures PHP Storm project (partially at the moment) + 1. Installs NodeJS, NPM, Grunt and Gulp for front end development + + :information_source: This box uses the [n package manager](https://www.npmjs.com/package/n) to provide the latest NodeJS LTS version.
## How to install -If you never used Vagrant before, read [Vagrant Docs](https://docs.vagrantup.com/v2) +If you never used Vagrant before, read the [Vagrant Docs](https://docs.vagrantup.com/v2) first. ### Requirements -Software listed below should be available in [PATH](https://en.wikipedia.org/wiki/PATH_\(variable\)) (except for PHP Storm). +The software listed below should be available in [PATH](https://en.wikipedia.org/wiki/PATH_\(variable\)) (except for PHP Storm). - [Vagrant 1.8+](https://www.vagrantup.com/downloads.html) - [VirtualBox](https://www.virtualbox.org/wiki/Downloads) -- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). Make sure you have SSH keys generated and associated with your github account, see [how to check](https://help.github.com/articles/testing-your-ssh-connection/) and [how to configure](https://help.github.com/articles/generating-ssh-keys/) if not configured.
-:information_source: It is possible to use another way of getting codebase instead of cloning, it does not matter for successful installation. Just put Magento 2 codebase inside of `vagrant-magento/magento2ce`.
-:information_source: ![](docs/images/windows-icon.png) On Windows hosts Git must be [v2.7+](http://git-scm.com/download/win), also make sure to set the following options to avoid issues with incorrect line separators: +- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - Ensure that SSH keys are generated and associated with your Github account. See [how to check](https://help.github.com/articles/testing-your-ssh-connection/) and [how to configure](https://help.github.com/articles/generating-ssh-keys/), if not configured.
+ :information_source: To obtain the codebase without cloning, just use the Magento 2 codebase instead of `vagrant-magento/magento2ce`. Either method will produce a successful installation.
+ + :information_source: On Windows hosts ![](docs/images/windows-icon.png) Git must be [v2.7+](http://git-scm.com/download/win). Also make sure to set the following options to avoid issues with incorrect line separators: ``` git config --global core.autocrlf false git config --global core.eol LF git config --global diff.renamelimit 5000 ``` -- ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) [PHP](http://php.net/manual/en/install.php) (any version) to allow Magento dependency management with [Composer](https://getcomposer.org/doc/00-intro.md) -- [PHP Storm](https://www.jetbrains.com/phpstorm) is optional but recommended. -- ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) [NFS server](https://en.wikipedia.org/wiki/Network_File_System) must be installed and running on \*nix and OSX hosts. Is usually available, so just try to follow [installation steps](#how-to-install) first. +- [PHP](http://php.net/manual/en/install.php) ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) (any version) to allow Magento dependency management with [Composer](https://getcomposer.org/doc/00-intro.md) +- [PHP Storm](https://www.jetbrains.com/phpstorm), optional but recommended +- [NFS server](https://en.wikipedia.org/wiki/Network_File_System) ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) must be installed and running on \*nix and OSX hosts; usually available, follow [installation steps](#how-to-install) first ### Installation steps - - 1. Open terminal and change directory to the one which you want to contain Magento project. ![](docs/images/windows-icon.png) On Windows use Git Bash, which is available after Git installation - 1. Download project with Vagrant configuration: - - ``` - git clone git@github.com:paliarush/magento2-vagrant-for-developers.git vagrant-magento - ``` - - 1. Copy [etc/composer/auth.json.dist](etc/composer/auth.json.dist) to `etc/composer/auth.json` and specify your [GitHub OAuth token](https://github.com/settings/tokens) there. See [API rate limit and OAuth tokens](https://getcomposer.org/doc/articles/troubleshooting.md#api-rate-limit-and-oauth-tokens) for more information - - 1. Optionally, copy [etc/config.yaml.dist](etc/config.yaml.dist) as `etc/config.yaml` and make necessary customizations - - 1. Initialize project, configure environment, install Magento, configure PHPStorm project: - - ``` - cd vagrant-magento - bash init_project.sh - ``` - - :information_source: ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) On OSX and \*nix hosts NFS will be used by default to sync your project files with guest. On some hosts Vagrant cannot configure NFS properly, in this case it is possible to deploy project without NFS by setting `use_nfs` option in [config.yaml](etc/config.yaml.dist) to `0`
- :information_source: ![](docs/images/windows-icon.png) On Windows hosts you might face `Composer Install Error: ZipArchive::extractTo(): Full extraction path exceed MAXPATHLEN (260)` exception during `composer install`. This can be fixed in 2 ways: decrease path length to the project directory or set `composer_prefer_source` option in [config.yaml](etc/config.yaml.dist) to `1` +:information_source: In case of any issues during installation, please read [FAQ section](#faq) + + 1. Open terminal and change your directory to the one you want to contain Magento project. ![](docs/images/windows-icon.png) On Windows use Git Bash, which is available after Git installation. + + 1. Download or clone the project with Vagrant configuration: + + :warning: Do not open it in PhpStorm until `init_project.sh` has completed PhpStorm configuration in the initialize project step below. + + ``` + git clone git@github.com:paliarush/magento2-vagrant-for-developers.git vagrant-magento + ``` + + Optionally, if you use private repositories on GitHub or download packages from the Magento Marketplace using Composer + + 1. Copy [etc/composer/auth.json.dist](etc/composer/auth.json.dist) to `etc/composer/auth.json`. + 1. Specify your GitHub token by adding `"github.com": "your-github-token"` to the `github-oauth` section for GitHub authorization. + 1. Add the Magento Marketplace keys for Marketplace authorization to the `repo.magento.com` section. + 1. Copy (optional) [etc/config.yaml.dist](etc/config.yaml.dist) as `etc/config.yaml` and make the necessary customizations. + + 1. Initialize the project (this will configure the environment, install Magento, and configure the PHPStorm project): - 1. Use `vagrant-magento` directory as project root in PHP Storm (not `vagrant-magento/magento2ce`). This is important, because in this case PHP Storm will be configured automatically by [init_project.sh](init_project.sh). If NFS files sync is disabled in [config](etc/config.yaml.dist) and ![](docs/images/windows-icon.png) on Windows hosts [verify deployment configuration in PHP Storm](docs/phpstorm-configuration-windows-hosts.md) + ``` + cd vagrant-magento + bash init_project.sh + ``` + + 1. Use the `vagrant-magento` directory as the project root in PHP Storm (not `vagrant-magento/magento2ce`). This is important, because in this case PHP Storm will be configured automatically by [init_project.sh](init_project.sh). If NFS files sync is disabled in [config](etc/config.yaml.dist) and ![](docs/images/windows-icon.png)on Windows hosts [verify the deployment configuration in PHP Storm](docs/phpstorm-configuration-windows-hosts.md). + + Use the URL for accessing your Magento storefront in the browser as your Web server root URL. Typically this is the localhost, which refers to your development machine. Depending on how you've set up your VM you may also need a port number, like `http://localhost:8080`. + + 1. Configure the remote PHP interpreter in PHP Storm. Go to Preferences, then Languages and Frameworks. Click PHP and add a new remote interpreter. Select Deployment configuration as a source for connection details. ### Default credentials and settings Some of default settings are available for override. These settings can be found in the file [etc/config.yaml.dist](etc/config.yaml.dist). -To override settings just create a copy of the file under the name 'config.yaml' and put there your custom settings. -When using [init_project.sh](init_project.sh), if not specified manually, random IP address is generated and is used as suffix for host name to prevent collisions, in case when 2 or more instances are running at the same time. + +To override settings create a copy of the file under the name 'config.yaml' and add your custom settings. + +When using [init_project.sh](init_project.sh), if not specified manually, random IP address is generated and is used as suffix for host name to prevent collisions, in case when two or more instances are running at the same time. + Upon a successful installation, you'll see the location and URL of the newly-installed Magento 2 application in console. **Web access**: @@ -98,6 +129,8 @@ Upon a successful installation, you'll see the location and URL of the newly-ins - Magento admin user/password: `admin/123123q` - Rabbit MQ control panel: `http://magento2.vagrant:15672`, credentials `guest`/`guest` +:information_source: Your admin URL, storefront URL, and admin user and password are located in `etc/config.yaml.dist`. + **Codebase and DB access**: - Path to your Magento installation on the VM: - Can be retrieved from environment variable: `echo ${MAGENTO_ROOT}` @@ -105,7 +138,7 @@ Upon a successful installation, you'll see the location and URL of the newly-ins - ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) On Mac and \*nix hosts: the same as on host - MySQL DB host: `localhost` (not accessible remotely) - MySQL DB name: `magento`, `magento_integration_tests` -- MySQL DB user/password: just use `mysql` with no user and password (`root/password` will be used by default) +- MySQL DB user/password: `root:`. In CLI just use `mysql` with no user and password (`root:` will be used by default) **Codebase on host** - CE codebase: `vagrant_project_root/magento2ce` @@ -117,31 +150,25 @@ Current vagrant project follows [semantic versioning](http://semver.org/spec/v2. For example your current branch is `2.0`, then it will be safe to pull any changes from `origin/2.0`. However branch `3.0` will contain changes backward incompatible with `2.0`. Note, that semantic versioning is only used for `x.0` branches (not for `develop`). -### GitHub limitations +:information_source: To apply changes run `vagrant reload`. -Be aware that you may encounter GitHub limits on the number of downloads (used by Composer to download Magento dependencies). +## Day-to-day development scenarios -These limits may significantly slow down the installation since all of the libraries will be cloned from GitHub repositories instead of downloaded as ZIP archives. In the worst case, these limitations may even terminate the installation. +### Reinstall Magento -If you have a GitHub account, you can bypass these limitations by using an OAuth token in the Composer configuration. See [API rate limit and OAuth tokens](https://getcomposer.org/doc/articles/troubleshooting.md#api-rate-limit-and-oauth-tokens) for more information. +Use commands described in [Switch between CE and EE](#switch-between-ce-and-ee) section with `-f` flag. Before doing actual re-installation, these commands update linking of EE codebase, clear cache, update composer dependencies. -For the Vagrant configuration you may specify your token in `etc/github.oauth.token` file after cloning the repository. The file is a basic text file and is ignored by Git, so you'll need to create it yourself. Simply write your OAuth token in this file making sure to avoid any empty spaces, and it will be read during deployment. +If no composer update and relinking of EE codebase is necessary, use the following command. It will clear Magento DB, Magento caches and reinstall Magento instance. -## Day-to-day development scenarios - -### Reinstall Magento -To save some time and get clear Magento installation, you can skip installation of software like web server or php. -The following command will clear Magento DB, Magento caches and reinstall Magento instance. - -Go to 'vagrant-magento' created earlier and run in command line: +Go to the root of vagrant project in command line and execute: ``` bash m-reinstall ``` -### Clear magento cache +### Clear Magento cache -Go to 'vagrant-magento' created earlier and run in command line: +Go to the root of vagrant project in command line and execute: ``` bash m-clear-cache @@ -159,25 +186,195 @@ OR bash m-switch-to-ee ``` -### Update composer dependencies +Force switch can be done using `-f` flag even if already switched to the target edition. May be helpful to relink EE modules after switching between branches. + +Upgrade can be performed instead of re-installation using `-u` flag. + +:information_source: On Windows hosts (or when NFS mode is disabled in [config.yaml](etc/config.yaml.dist) explicitly) you will be asked to wait until code is uploaded to guest machine by PhpStorm (PhpStorm must be launched). To continue the process press any key. + +### Sample data installation + +Make sure that `ce_sample_data` and `ee_sample_data` are defined in [config.yaml](etc/config.yaml.dist) and point CE and optionally EE sample data repositories. +During initial project setup or during `bash init_project.sh -fc` (with `-fc` project will be re-created from scratch), sample data repositories willl be checked out to `vagrant_project_root/magento2ce/magento2ce-sample-data` and `vagrant_project_root/magento2ce/magento2ee-sample-data`. + +To install Magento with sample data set `install_sample_data` in [config.yaml](etc/config.yaml.dist) to `1` and run `bash m-switch-to-ce -f` or `bash m-switch-to-ee -f`, depending on the edition to be installed. To disable sample data, set this option to `0` and force-switch to necessary edition (using the same commands). + +### Basic data generation + +Several entities are generated for testing purposes by default using REST API after Magento installation: +- Customer with address (credentials `customer@example.com`:`123123qQ`) +- Category +- Couple simple products +- Configurable product + +To disable this feature, set `magento/generate_basic_data` in [config.yaml](etc/config.yaml.dist) to `0` and run `bash m-switch-to-ce -f` or `bash m-switch-to-ee -f`, depending on the edition to be installed. + +### Use Magento CLI (bin/magento) Go to 'vagrant-magento' created earlier and run in command line: ``` -bash m-composer install -OR -bash m-composer update +bash m-bin-magento +e.g. +bash m-bin-magento list ``` ### Debugging with XDebug XDebug is already configured to connect to the host machine automatically. So just: - + 1. Set XDEBUG_SESSION=1 cookie (e.g. using 'easy Xdebug' extension for Firefox). See [XDebug documentation](http://xdebug.org/docs/remote) for more details 1. Start listening for PHP Debug connections in PhpStorm on default 9000 port. See how to [integrate XDebug with PhpStorm](https://www.jetbrains.com/phpstorm/help/configuring-xdebug.html#integrationWithProduct) 1. Set beakpoint or set option in PhpStorm menu 'Run -> Break at first line in PHP scripts' - + +To debug a CLI script: + + 1. Create [remote debug configuration](https://www.jetbrains.com/help/phpstorm/2016.1/run-debug-configuration-php-remote-debug.html) in PhpStorm, use `phpstorm` as IDE key + 1. Run created remote debug configuration + 1. Run CLI command on the guest as follows (`xdebug.remote_host` value might be different for you): + + ``` + php -d xdebug.remote_autostart=1 + ``` + +To debug Magento Setup script, go to [Magento installation script](scripts/guest/m-reinstall) and find `php ${install_cmd}`. Follow steps above for any CLI script + +:information_source: In addition to XDebug support, [config.yaml](etc/config.yaml.dist) has several options in `debug` section which allow storefront and admin UI debugging. Plus, desired Magento mode (developer/production/default) can be enabled using `magento_mode` option, default is developer mode. + +### Connecting to MySQL DB + +Answer can be found [here](https://github.com/paliarush/magento2-vagrant-for-developers/issues/8) + +### View emails sent by Magento + +All emails are saved to 'vagrant-magento/log/email' in HTML format. + +### Accessing PHP and other config files + +It is possible to view/modify majority of guest machine config files directly from IDE on the host. They will be accessible in [etc/guest](etc/guest) directory only when guest machine is running. The list of accessible configs includes: PHP, Apache, Mysql, Varnish, RabbitMQ. +Do not edit any symlinks using PhpStorm because it may break your installation. + +After editing configs in IDE it is still required to restart related services manually. + +### Upgrading Magento + +Sometimes it is necessary to test upgrade flow. This can be easily done as follows (assuming that you have installed instance): + + - For git-based installation - check out codebase corresponding to the target Magento version. Or modify your `composer.json` in case of composer-based installation + - Use commands described in [Switch between CE and EE](#switch-between-ce-and-ee) section with `-u` flag + ### Multiple Magento instances - + To install several Magento instances based on different code bases, just follow [Installation steps](#installation-steps) to initialize project in another directory on the host. Unique IP address, SSH port and domain name will be generated for each new instance if not specified manually in `etc/config.yaml` + +### Update Composer dependencies + +Go to 'vagrant-magento' created earlier and run in command line: + +``` +bash m-composer install +OR +bash m-composer update +``` + +### Running Magento tests + +See [draft](https://github.com/paliarush/magento2-vagrant-for-developers/issues/120) + + +## Environment configuration + +### Switch between PHP versions + +Switch between PHP versions using "php_version: " option in [config.yaml](etc/config.yaml.dist). Supported versions are 5.6, 7.0, 7.1 and 7.2. +PHP version will be applied after "vagrant reload". + +### Activating Varnish + +Set `use_varnish: 1` to use varnish along with apache in [config.yaml](etc/config.yaml.dist). Changes will be applied on `m-reinstall`. +It will use default file etc/magento2_default_varnish.vcl.dist generated from a Magento 2.1 instance. Magento 2.2+ supports additional Varnish features and you may need to provide custom version of VCL to enable them. +Varnish Version: 4.1 + +Use the following commands to enable/disable varnish without reinstalling Magento: `m-varnish disable` or `m-varnish enable`. + +### Activating ElasticSearch + +:information_source: Available in Magento EE only. + +Set `search_engine: "elasticsearch"` in [config.yaml](etc/config.yaml.dist) to use ElasticSearch as current search engine or `search_engine: "mysql"` to use MySQL. Changes will be applied on `m-reinstall`. + +Use the following commands to switch between search engines without reinstalling Magento: `m-search-engine elasticsearch` or `m-search-engine mysql`. + +### Redis for caching + +:information_source: Available in Magento v2.0.6 and higher. + +Redis is configured as cache backend by default. It is still possible to switch back to filesystem cache by changing `environment_cache_backend` to `filesystem` in [config.yaml](etc/config.yaml.dist). + +### Reset environment + +It is possible to reset project environment to default state, which you usually get just after project initialization. The following command will delete vagrant box and vagrant project settings. After that it will initialize project from scratch. Magento 2 code base (`magento2ce` directory) and [etc/config.yaml](etc/config.yaml.dist) and PhpStorm settings will stay untouched, but guest config files (located in [etc/guest](etc/guest)) will be cleared. + +Go to 'vagrant-magento' created earlier and run in command line: + +``` +bash init_project.sh -f +``` + +It is possible to reset Magento 2 code base at the same time. Magento 2 code base will be deleted and then cloned from the repositories specified in [etc/config.yaml](etc/config.yaml.dist) + +``` +bash init_project.sh -fc +``` + +To reset PhpStorm project configuration, in addition to `-f` specify `-p` option: + +``` +bash init_project.sh -fp +``` + +Ultimate project reset can be achieved by combining all available flags: + +``` +bash init_project.sh -fcp +``` + +### Switch NodeJS Versions + +By default, the box will install the latest `NodeJS LTS` version using the [n package manager](https://www.npmjs.com/package/n). If you need another version of `Node` because of Magento's `package.json` requirements, simply run: + +```js +n +``` + +Note: See [Working with npm](https://www.npmjs.com/package/n#working-with-npm) if after switching versions with `n`, `npm` is not working properly. + +### FAQ + + 1. To debug any CLI script in current Vagrant project, set `debug:vagrant_project` option in [config.yaml](etc/config.yaml.dist) to `1` + 1. Is Windows 10 supported? Yes, but you may face the same issue as described [here](https://github.com/paliarush/magento2-vagrant-for-developers/issues/36) or [here](https://github.com/paliarush/magento2-vagrant-for-developers/issues/173). Also Virtual box may not work on Windows 10 in headless mode, see how to [enable GUI mode](https://www.vagrantup.com/docs/virtualbox/configuration.html) + 1. ![](docs/images/linux-icon.png)![](docs/images/osx-icon.png) On OSX and \*nix hosts NFS will be used by default to sync your project files with guest. On some hosts Vagrant cannot configure NFS properly, in this case it is possible to deploy project without NFS by setting `use_nfs` option in [config.yaml](etc/config.yaml.dist) to `0`
+ 1. ![](docs/images/windows-icon.png) On Windows hosts you might face `Composer Install Error: ZipArchive::extractTo(): Full extraction path exceed MAXPATHLEN (260)` exception during `composer install`. This can be fixed in 2 ways: decrease path length to the project directory or set `composer_prefer_source` option in [config.yaml](etc/config.yaml.dist) to `1` + 1. Make sure that you used `vagrant-magento` directory as project root in PHP Storm (not `vagrant-magento/magento2ce`) + 1. If project opened in PhpStorm looks broken, close PhpStorm and remove `vagrant-magento/.idea`. Run `bash vagrant-magento/scripts/host/configure_php_storm.sh`. After opening project in PhpStorm again everything should look good + 1. If code is not synchronized properly on Windows hosts (or when NFS mode is disabled in [config.yaml](etc/config.yaml.dist) explicitly), make sure that PhpStorm is running before making any changes in the code. This is important because otherwise PhpStorm will not be able to detect changes and upload them to the guest machine + 1. Please make sure that currently installed software, specified in [requirements section](#requirements), meets minimum version requirement + 1. Be careful if your OS is case-insensitive, NFS might break the symlinks if you cd into the wrong casing and you power the vagrant up. Just be sure to cd in to the casing the directory was originally created as. + 1. Cannot run unit tests from PHPStorm on Magento 2.2, see possible solution [here](https://github.com/paliarush/magento2-vagrant-for-developers/issues/167) + 1. [Permission denied (publickey)](https://github.com/paliarush/magento2-vagrant-for-developers/issues/165) + 1. If during a vagrant reload, the following message appears: + + >There was a problem while downloading the metadata for your box + to check for updates. This is not an error, since it is usually due + to temporary network problems. This is just a warning. The problem + encountered was: + The requested URL returned error: 404 Not Found + + It is likely that your vagrant cli is caching an old url. Perform the following cli commands: + + ```bash + sed -i -- 's/atlas.hashicorp/vagrantcloud/g' ~/.vagrant.d/boxes/{name of your paliarush/ubuntu image}/metadata_url + mv ~/.vagrant.d/boxes/{name of your paliarush/ubuntu image}/metadata_url2 ~/.vagrant.d/boxes/{name of your paliarush/ubuntu image}/metadata_url + ``` + diff --git a/Vagrantfile b/Vagrantfile index 8d8ea24cb1e..e7fdfa3a18d 100755 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,9 +1,10 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.require_version "~> 1.8" +Vagrant.require_version ">= 1.8" require 'yaml' +require 'vagrant/util/deep_merge' module OS def OS.is_windows @@ -20,7 +21,7 @@ module Config config_data_dist = YAML.load_file(config_dist_file) config_data = File.exists?(config_file) ? YAML.load_file(config_file) : {} - return config_data_dist.merge!(config_data) + return Vagrant::Util::DeepMerge.deep_merge(config_data_dist, config_data) end end @@ -30,55 +31,83 @@ magento_host_name = config_data['magento']['host_name'] magento_ip_address = config_data['guest']['ip_address'] forwarded_ssh_port = config_data['guest']['forwarded_ssh_port'] guest_memory = config_data['guest']['memory'] +guest_cpus = config_data['guest']['cpus'] # NFS will be used for *nix and OSX hosts, if not disabled explicitly in config use_nfs_for_synced_folders = !OS.is_windows && (config_data['guest']['use_nfs'] == 1) -host_magento_dir = Dir.pwd + '/magento2ce' +host_vagrant_dir = Dir.pwd + '' +host_magento_dir = host_vagrant_dir + '/magento2ce' VAGRANT_API_VERSION = 2 Vagrant.configure(VAGRANT_API_VERSION) do |config| config.vm.box = "paliarush/magento2.ubuntu" - config.vm.box_version = "~> 1.0" + config.vm.box_version = "~> 1.1" config.vm.provider "virtualbox" do |vb| vb.memory = guest_memory + vb.cpus = guest_cpus + # Uncomment option below to avoid issues with VirtualBox on Windows 10 + # vb.gui=true end config.vm.synced_folder '.', '/vagrant', disabled: true - config.vm.synced_folder './etc', '/vagrant/etc' + config.vm.synced_folder './etc', '/vagrant/etc', mount_options: ["dmode=775,fmode=664"] config.vm.synced_folder './scripts', '/vagrant/scripts' + config.vm.synced_folder './log', '/vagrant/log' config.vm.synced_folder './.idea', '/vagrant/.idea', create: true if use_nfs_for_synced_folders guest_magento_dir = host_magento_dir config.vm.synced_folder host_magento_dir, guest_magento_dir, type: "nfs", create: true else guest_magento_dir = '/var/www/magento2ce' - config.vm.synced_folder host_magento_dir + '/var/generation', guest_magento_dir + '/var/generation', create: true + config.vm.synced_folder host_magento_dir + '/var', guest_magento_dir + '/var', create: true + config.vm.synced_folder host_magento_dir + '/generated', guest_magento_dir + '/generated', create: true config.vm.synced_folder host_magento_dir + '/app/etc', guest_magento_dir + '/app/etc', create: true end shell_script_args = [ - use_nfs_for_synced_folders ? "1" : "0", #1 - guest_magento_dir, #2 - magento_host_name, #3 - config_data['environment']['use_php7'], #4 - host_magento_dir #5 + use_nfs_for_synced_folders ? "1" : "0", #1 + guest_magento_dir, #2 + magento_host_name, #3 + config_data['environment']['use_php7'] || 0, #4 TODO: Remove legacy parameter, replaced with php_version + host_magento_dir, #5 + OS.is_windows ? "1" : "0", #6 + host_vagrant_dir, #7 + config_data['environment']['php_version'] #8 ] - config.vm.provision "configure_environment", type: "shell" do |s| + config.vm.provision "fix_no_tty", type: "shell", run: "always" do |s| + s.privileged = false + s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile" + end + + config.vm.provision "upgrade_environment_recurring", type: "shell", run: "always", keep_color: true do |s| + s.path = "scripts/provision/upgrade_environment_recurring.sh" + s.args = shell_script_args + end + + config.vm.provision "configure_environment", type: "shell", keep_color: true do |s| s.path = "scripts/provision/configure_environment.sh" s.args = shell_script_args end + config.vm.provision "configure_environment_recurring", type: "shell", run: "always", keep_color: true do |s| + s.path = "scripts/provision/configure_environment_recurring.sh" + s.args = shell_script_args + end + + config.vm.provision "export_env_variables_recurring", type: "shell", run: "always", keep_color: true do |s| + s.path = "scripts/provision/export_env_variables_recurring.sh" + s.args = shell_script_args + end + if !use_nfs_for_synced_folders config.vm.provision "host_compress_magento_code", type: "host_shell", inline: "tar cf scripts/host/magento2ce.tar magento2ce" config.vm.provision "guest_uncompress_magento_code", type: "shell", inline: "mkdir -p /var/www && tar xf /vagrant/scripts/host/magento2ce.tar -C /var/www &>/dev/null" config.vm.provision "guest_remove_compressed_code", type: "shell", inline: "rm -f /vagrant/scripts/host/magento2ce.tar" end - config.vm.provision "install_magento", type: "shell", inline: "m-reinstall" - # Host manager plugin configuration config.hostmanager.enabled = true config.hostmanager.manage_host = true diff --git a/docs/images/release_badge.png b/docs/images/release_badge.png index 73ab6b65214..284d80c3dc7 100644 Binary files a/docs/images/release_badge.png and b/docs/images/release_badge.png differ diff --git a/docs/images/release_badge.svg b/docs/images/release_badge.svg index 3d7870857ab..34be2a956c9 100644 --- a/docs/images/release_badge.svg +++ b/docs/images/release_badge.svg @@ -1 +1 @@ -releasereleasev2.0.0v2.0.0 \ No newline at end of file +releasereleasev2.2.0v2.2.0 diff --git a/docs/images/tests_badge.png b/docs/images/tests_badge.png new file mode 100644 index 00000000000..ad1430e79e4 Binary files /dev/null and b/docs/images/tests_badge.png differ diff --git a/docs/images/tests_badge.svg b/docs/images/tests_badge.svg new file mode 100644 index 00000000000..f8d79906345 --- /dev/null +++ b/docs/images/tests_badge.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + tests + tests + passing on OSX + passing on OSX + + diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 00000000000..0bb141367b2 --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,10 @@ +Known issues +---------------------------- + +1. PHP Warning: require(/var/www/magento2ce/app/etc/NonComposerComponentRegistration.php): failed to open stream: No such file or directory + Check modified files from /var/www/magento2ce/app/etc/ in git and revert them. + +2. Guest system behaves wierd, e.g. install process can be suddenly killed. + Try to increase memory for guest machine. + + diff --git a/docs/phpstorm-configuration-windows-hosts.md b/docs/phpstorm-configuration-windows-hosts.md index 29b252dfc95..de4e6fed083 100644 --- a/docs/phpstorm-configuration-windows-hosts.md +++ b/docs/phpstorm-configuration-windows-hosts.md @@ -7,7 +7,7 @@ This solution is applicable to any Vagrant configuration for Magento instance, s ``` Vagrant.configure(2) do |config| - config.vm.synced_folder '../magento2ce/var/generation', '/var/www/magento2ce/var/generation' + config.vm.synced_folder '../magento2ce/var', '/var/www/magento2ce/var' config.vm.synced_folder '../magento2ce/app/etc', '/var/www/magento2ce/app/etc' end ``` diff --git a/etc/.gitignore b/etc/.gitignore index c62aab67d1f..44c9e13080d 100644 --- a/etc/.gitignore +++ b/etc/.gitignore @@ -5,3 +5,4 @@ !/composer/auth.json.dist !/config.yaml.dist !/magento2_virtual_host.conf.dist +!/guest \ No newline at end of file diff --git a/etc/composer/auth.json.dist b/etc/composer/auth.json.dist index 1058d7070ba..024c557834c 100644 --- a/etc/composer/auth.json.dist +++ b/etc/composer/auth.json.dist @@ -1,6 +1,5 @@ { "github-oauth": { - "github.com": "your-github-token" }, "http-basic": { "repo.magento.com": { diff --git a/etc/config.yaml.dist b/etc/config.yaml.dist index 92dcd23c89c..b9b582c24e2 100644 --- a/etc/config.yaml.dist +++ b/etc/config.yaml.dist @@ -1,22 +1,70 @@ +# [To apply changes: init_project.sh -fc] Possible values: git, composer +checkout_source_from: "git" + +composer_project: + # Current section used only if 'checkout_source_from' is set to 'composer' + + # [To apply changes: init_project.sh -fc] + # Custom version can be provided by using '={version}' appended to repo name. E.g. name: "magento/project-community-edition=2.2.0". Otherwise, default version is used + name: "magento/project-community-edition" + # Make sure to specify valid keys for this repository in etc/composer/auth.json + url: "https://repo.magento.com" + repository_url: + # Current section used only if 'checkout_source_from' is set to 'git' + + # [To apply changes: init_project.sh -fc] + # git@github.com:magento/magento2ce.git OR custom one. Use '::' to specify a specific branch, e.g: git@github.com:magento/magento2.git::2.1 ce: "git@github.com:magento/magento2.git" + # git@github.com:magento/magento2ee.git OR custom one ee: "" + # can be replaced with custom repository. Use '::' to specify a specific branch, e.g: git@github.com:magento/magento2-sample-data.git::2.1-develop + ce_sample_data: "git@github.com:magento/magento2-sample-data.git" + # git@github.com:magento/magento2-sample-data-ee.git OR custom one + ee_sample_data: "" + guest: - # NFS will be used for folder synchronization on *nix and OSX hosts by default + # [To apply changes: init_project.sh -f] NFS will be used for folder synchronization on *nix and OSX hosts by default. use_nfs: 1 - # Default is 2Gb, around 3Gb is necessary to run functional tests + # [To apply changes: vagrant reload] Default is 2Gb, around 3Gb is necessary to run functional tests. memory: 2048 + # Recommended number of CPUs is 2 + cpus: 1 ip_address: "192.168.10.2" forwarded_ssh_port: 3000 + environment: - # If set to 0, PHP 5 will be installed - use_php7: 1 + # [To apply changes: vagrant reload] Valid versions: 5.6, 7.0, 7.1, 7.2 + php_version: "7.0" composer_prefer_source: 0 + # [To apply changes: m-reinstall OR m-varnish enable/disable] + use_varnish: 0 + # [To apply changes: m-reinstall] Possible values: mysql, elasticsearch + search_engine: "mysql" + # [To apply changes: m-clear-cache] Possible values: redis, filesystem + cache_backend: "redis" + magento: + # [To apply changes: m-switch-to-ce -f OR m-switch-to-ee -f] + install_sample_data: 0 + # [To apply changes: m-switch-to-ce -f OR m-switch-to-ee -f] Generate customer (customer@example.com:123123qQ), category, simple product, configurable product after successful Magento installation + generate_basic_data: 1 + # [To apply changes: init_project.sh -f] host_name: "magento2.vagrant2" + # [To apply changes: m-reinstall] admin_frontname: "admin" language: "en_US" timezone: "America/Chicago" currency: "USD" admin_user: "admin" admin_password: "123123q" + # [Changes applied on m-clear-cache] Available options: developer, production, default + mode: "developer" + +debug: + # [Changes applied on m-clear-cache] Enable template path and block name hints on storefront + magento_storefront: 0 + # [Changes applied on m-clear-cache] Enable template path hints in the admin panel + magento_admin: 0 + # [Changes applied immediately] Enable detailed output from all scripts provided by current vagrant project + vagrant_project: 0 diff --git a/etc/guest/.gitignore b/etc/guest/.gitignore new file mode 100644 index 00000000000..22e83649f7d --- /dev/null +++ b/etc/guest/.gitignore @@ -0,0 +1 @@ +/* \ No newline at end of file diff --git a/etc/magento2_default_varnish.vcl.dist b/etc/magento2_default_varnish.vcl.dist new file mode 100644 index 00000000000..ae934f7eba7 --- /dev/null +++ b/etc/magento2_default_varnish.vcl.dist @@ -0,0 +1,169 @@ +vcl 4.0; + +import std; +# The minimal Varnish version is 4.0 +# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https' + +backend default { + .host = "localhost"; + .port = "8080"; +} + +acl purge { + "localhost"; +} + +sub vcl_recv { + if (req.method == "PURGE") { + if (client.ip !~ purge) { + return (synth(405, "Method not allowed")); + } + if (!req.http.X-Magento-Tags-Pattern) { + return (synth(400, "X-Magento-Tags-Pattern header required")); + } + ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern); + return (synth(200, "Purged")); + } + + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + + # We only deal with GET and HEAD by default + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + # Bypass shopping cart, checkout and search requests + if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") { + return (pass); + } + + # normalize url in case of leading HTTP scheme and domain + set req.url = regsub(req.url, "^http[s]?://", ""); + + # collect all cookies + std.collect(req.http.Cookie); + + # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression + if (req.http.Accept-Encoding) { + if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") { + # No point in compressing these + unset req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") { + set req.http.Accept-Encoding = "deflate"; + } else { + # unkown algorithm + unset req.http.Accept-Encoding; + } + } + + # Remove Google gclid parameters to minimize the cache objects + set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA" + set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar" + set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz" + + # static files are always cacheable. remove SSL flag and cookie + if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") { + unset req.http.Https; + unset req.http.X-Forwarded-Proto; + unset req.http.Cookie; + } + + return (hash); +} + +sub vcl_hash { + if (req.http.cookie ~ "X-Magento-Vary=") { + hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1")); + } + + # For multi site configurations to not cache each other's content + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + + # To make sure http users don't see ssl warning + if (req.http.X-Forwarded-Proto) { + hash_data(req.http.X-Forwarded-Proto); + } + +} + +sub vcl_backend_response { + if (beresp.http.content-type ~ "text") { + set beresp.do_esi = true; + } + + if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") { + set beresp.do_gzip = true; + } + + # cache only successfully responses and 404s + if (beresp.status != 200 && beresp.status != 404) { + set beresp.ttl = 0s; + set beresp.uncacheable = true; + return (deliver); + } elsif (beresp.http.Cache-Control ~ "private") { + set beresp.uncacheable = true; + set beresp.ttl = 86400s; + return (deliver); + } + + if (beresp.http.X-Magento-Debug) { + set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control; + } + + # validate if we need to cache it and prevent from setting cookie + # images, css and js are cacheable by default so we have to remove cookie also + if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) { + unset beresp.http.set-cookie; + if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") { + set beresp.http.Pragma = "no-cache"; + set beresp.http.Expires = "-1"; + set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0"; + set beresp.grace = 1m; + } + } + + # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass + if (beresp.ttl <= 0s || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && beresp.http.Vary == "*")) { + # Mark as Hit-For-Pass for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + return (deliver); +} + +sub vcl_deliver { + if (resp.http.X-Magento-Debug) { + if (resp.http.x-varnish ~ " ") { + set resp.http.X-Magento-Cache-Debug = "HIT"; + } else { + set resp.http.X-Magento-Cache-Debug = "MISS"; + } + } else { + unset resp.http.Age; + } + + unset resp.http.X-Magento-Debug; + unset resp.http.X-Magento-Tags; + unset resp.http.X-Powered-By; + unset resp.http.Server; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Link; +} diff --git a/etc/magento2_virtual_host.conf.dist b/etc/magento2_virtual_host.conf.dist index 806292bc917..eebe0267f55 100755 --- a/etc/magento2_virtual_host.conf.dist +++ b/etc/magento2_virtual_host.conf.dist @@ -1,7 +1,7 @@ - DocumentRoot + DocumentRoot "" ServerName - > + "> Options Indexes FollowSymLinks AllowOverride All Require all granted diff --git a/init_project.sh b/init_project.sh index ba225a25f76..fab1c2155d6 100755 --- a/init_project.sh +++ b/init_project.sh @@ -1,18 +1,152 @@ #!/usr/bin/env bash +set -e + vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +config_path="${vagrant_dir}/etc/config.yaml" +if [[ ! -f "${config_path}" ]]; then + status "Initializing etc/config.yaml using defaults from etc/config.yaml.dist" + cp "${config_path}.dist" "${config_path}" +fi + magento_ce_dir="${vagrant_dir}/magento2ce" +magento_ce_sample_data_dir="${magento_ce_dir}/magento2ce-sample-data" magento_ee_dir="${magento_ce_dir}/magento2ee" -config_path="${vagrant_dir}/etc/config.yaml" -host_os=$(bash "${vagrant_dir}/scripts/host/get_host_os.sh") +magento_ee_sample_data_dir="${magento_ce_dir}/magento2ee-sample-data" +host_os="$(bash "${vagrant_dir}/scripts/host/get_host_os.sh")" +use_nfs="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs")" +repository_url_ce="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ce")" +repository_url_ee="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ee")" +composer_project_name="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "composer_project_name")" +composer_project_url="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "composer_project_url")" +checkout_source_from="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "checkout_source_from")" + +function checkoutSourceCodeFromGit() +{ + if [[ ! -d ${magento_ce_dir} ]]; then + if [[ ${host_os} == "Windows" ]]; then + status "Configuring git for Windows host" + git config --global core.autocrlf false + git config --global core.eol LF + git config --global diff.renamelimit 5000 + fi + + initMagentoCeGit + initMagentoCeSampleGit + + # By default EE repository is not specified and EE project is not checked out + if [[ -n "${repository_url_ee}" ]]; then + initMagentoEeGit + fi + # By default EE sample data repository is not specified and EE project is not checked out + repository_url_ee_sample_data="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ee_sample_data")" + if [ -n "${repository_url_ee_sample_data}" ]; then + initMagentoEeSampleGit + fi + fi +} + +function initMagentoCeGit() +{ + initGitRepository ${repository_url_ce} "CE" "${magento_ce_dir}" +} + +function initMagentoEeGit() +{ + initGitRepository ${repository_url_ee} "EE" "${magento_ee_dir}" +} + +function initMagentoCeSampleGit() +{ + repository_url_ce_sample_data="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ce_sample_data")" + initGitRepository ${repository_url_ce_sample_data} "CE sample data" "${magento_ce_sample_data_dir}" +} + +function initMagentoEeSampleGit() +{ + initGitRepository ${repository_url_ee_sample_data} "EE sample data" "${magento_ee_sample_data_dir}" +} + +# Initialize the cloning and checkout of a git repository +# Arguments: +# Url of repository +# Name of repository (CE, EE) +# Directory where the repository will be cloned to +function initGitRepository() +{ + local repository_url=${1} + local repository_name=${2} + local directory=${3} + + if [[ ${repository_url} == *"::"* ]]; then + local branch=$(getGitBranch ${repository_url}) + local repo=$(getGitRepository ${repository_url}) + else + local repo=${repository_url} + fi -# Enable trace printing and exit on the first error -set -ex + status "Checking out ${repository_name} repository" + git clone ${repo} "${directory}" 2> >(logError) > >(log) + + if [[ -n ${branch} ]]; then + status "Checking out branch ${branch} of ${repository_name} repository" + cd "${directory}" + git fetch 2> >(logError) > >(log) + git checkout ${branch} 2> >(log) > >(log) + fi + cd "${vagrant_dir}" +} + +# Get the git repository from a repository_url setting in config.yaml +function getGitRepository() +{ + local repo="${1%::*}" # Gets the substring before the '::' characters + echo ${repo} +} + +# Get the git branch from a repository_url setting in config.yaml +function getGitBranch() +{ + local branch="${1#*::}" # Gets the substring after the '::' characters + echo ${branch} +} + +function composerCreateProject() +{ + if [[ ! -d ${magento_ce_dir} ]]; then + status "Downloading Magento codebase using 'composer create-project'" + bash "${vagrant_dir}/scripts/host/composer.sh" create-project ${composer_project_name} "${magento_ce_dir}" --repository-url=${composer_project_url} + + # TODO: Workaround for Magento 2.2+ until PHP is upgraded to 7.1 on the guest + cd "${magento_ce_dir}" + composer_dir="${vagrant_dir}/scripts/host" + composer_phar="${composer_dir}/composer.phar" + php_executable="$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh")" + project_version="$("${php_executable}" "${composer_phar}" show --self | grep version)" + matching_version_pattern='2.[23].[0-9]+' + if [[ ${project_version} =~ ${matching_version_pattern} ]]; then + status "Composer require zendframework/zend-code:~3.1.0 (needed for Magento 2.2+ only)" + cd "${magento_ce_dir}" + bash "${vagrant_dir}/scripts/host/composer.sh" require "zendframework/zend-code:~3.1.0" + fi + fi +} bash "${vagrant_dir}/scripts/host/check_requirements.sh" -# Install necessary vagrant plugins if not installed -vagrant_plugin_list=$(vagrant plugin list) +status "Installing missing vagrant plugins" +vagrant_plugin_list="$(vagrant plugin list)" if ! echo ${vagrant_plugin_list} | grep -q 'vagrant-hostmanager' ; then vagrant plugin install vagrant-hostmanager fi @@ -23,12 +157,9 @@ if ! echo ${vagrant_plugin_list} | grep -q 'vagrant-host-shell' ; then vagrant plugin install vagrant-host-shell fi -# Generate random IP address and host name to prevent collisions, if not specified explicitly in local config -if [ ! -f "${vagrant_dir}/etc/config.yaml" ]; then - cp "${config_path}.dist" ${config_path} -fi -random_ip=$(( ( RANDOM % 240 ) + 12 )) -forwarded_ssh_port=$(( random_ip + 3000 )) +status "Generating random IP address, and host name to prevent collisions (if no custom values specified)" +random_ip="$(( ( RANDOM % 240 ) + 12 ))" +forwarded_ssh_port="$(( random_ip + 3000 ))" sed -i.back "s|ip_address: \"192.168.10.2\"|ip_address: \"192.168.10.${random_ip}\"|g" "${config_path}" sed -i.back "s|host_name: \"magento2.vagrant2\"|host_name: \"magento2.vagrant${random_ip}\"|g" "${config_path}" sed -i.back "s|forwarded_ssh_port: 3000|forwarded_ssh_port: ${forwarded_ssh_port}|g" "${config_path}" @@ -37,44 +168,91 @@ rm -f "${config_path}.back" # Clean up the project before initialization if "-f" option was specified. Remove codebase if "-fc" is used. force_project_cleaning=0 force_codebase_cleaning=0 -while getopts 'fc' flag; do +force_phpstorm_config_cleaning=0 +while getopts 'fcp' flag; do case "${flag}" in f) force_project_cleaning=1 ;; c) force_codebase_cleaning=1 ;; - *) error "Unexpected option ${flag}" ;; + p) force_phpstorm_config_cleaning=1 ;; + *) error "Unexpected option" && exit 1;; esac done -if [ ${force_project_cleaning} -eq 1 ]; then - vagrant destroy -f - rm -rf ${vagrant_dir}/.idea ${vagrant_dir}/.vagrant - if [ ${force_codebase_cleaning} -eq 1 ]; then - rm -rf ${magento_ce_dir} +if [[ ${force_project_cleaning} -eq 1 ]]; then + status "Cleaning up the project before initialization since '-f' option was used" + vagrant destroy -f 2> >(logError) > >(log) + mv "${vagrant_dir}/etc/guest/.gitignore" "${vagrant_dir}/etc/.gitignore.back" + rm -rf "${vagrant_dir}/.vagrant" "${vagrant_dir}/etc/guest" + mkdir "${vagrant_dir}/etc/guest" + mv "${vagrant_dir}/etc/.gitignore.back" "${vagrant_dir}/etc/guest/.gitignore" + cd "${vagrant_dir}/log" && mv email/.gitignore email_gitignore.back && rm -rf email && mkdir email && mv email_gitignore.back email/.gitignore + if [[ ${force_codebase_cleaning} -eq 1 ]]; then + status "Removing current Magento codebase before initialization since '-c' option was used" + rm -rf "${magento_ce_dir}" fi fi -if [ ! -d ${magento_ce_dir} ]; then - if [[ ${host_os} == "Windows" ]]; then - git config --global core.autocrlf false - git config --global core.eol LF - git config --global diff.renamelimit 5000 +if [[ ! -d ${magento_ce_dir} ]]; then + if [[ "${checkout_source_from}" == "composer" ]]; then + composerCreateProject + elif [[ "${checkout_source_from}" == "git" ]]; then + checkoutSourceCodeFromGit + else + error "Value specified for 'checkout_source_from' is invalid. Supported options: composer OR git" + exit 1 fi - # Check out CE repository - repository_url_ce=$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ce") - git clone ${repository_url_ce} ${magento_ce_dir} - # Check out EE repository - # By default EE repository is not specified and EE project is not checked out - repository_url_ee=$(bash "${vagrant_dir}/scripts/get_config_value.sh" "repository_url_ee") - if [ -n "${repository_url_ee}" ]; then - git clone ${repository_url_ee} ${magento_ee_dir} +fi + +if [[ "${checkout_source_from}" == "git" ]]; then + status "Installing Magento dependencies via Composer" + cd "${magento_ce_dir}" + bash "${vagrant_dir}/scripts/host/composer.sh" install +fi + +status "Initializing vagrant box" +cd "${vagrant_dir}" + +vagrant up --provider virtualbox 2> >(logError) | { + while IFS= read -r line + do + filterVagrantOutput "${line}" + lastline="${line}" + done + filterVagrantOutput "${lastline}" +} + +bash "${vagrant_dir}/scripts/host/check_mounted_directories.sh" + +if [[ ${force_project_cleaning} -eq 1 ]] && [[ ${force_phpstorm_config_cleaning} -eq 1 ]]; then + status "Resetting PhpStorm configuration since '-p' option was used" + rm -rf "${vagrant_dir}/.idea" +fi +if [[ ! -f "${vagrant_dir}/.idea/deployment.xml" ]]; then + bash "${vagrant_dir}/scripts/host/configure_php_storm.sh" +fi +bash "${vagrant_dir}/scripts/host/configure_tests.sh" + +if [[ ${host_os} == "Windows" ]] || [[ ${use_nfs} == 0 ]]; then + # Automatic switch to EE during project initialization cannot be supported on Windows + status "Installing Magento CE" + bash "${vagrant_dir}/scripts/host/m_reinstall.sh" 2> >(logError) +else + if [[ -n "${repository_url_ee}" ]]; then + bash "${vagrant_dir}/scripts/host/m_switch_to_ee.sh" -f 2> >(logError) + else + bash "${vagrant_dir}/scripts/host/m_switch_to_ce.sh" -f 2> >(logError) fi fi -# Update Magento dependencies via Composer -cd ${magento_ce_dir} -bash "${vagrant_dir}/scripts/host/composer.sh" install +success "Project initialization succesfully completed (make sure there are no errors in the log above)" -# Create vagrant project -cd ${vagrant_dir} -vagrant up +info "$(bold)[Important]$(regular) + Please use $(bold)${vagrant_dir}$(regular) directory as PhpStorm project root, NOT $(bold)${magento_ce_dir}$(regular)." + +if [[ ${host_os} == "Windows" ]] || [[ ${use_nfs} == 0 ]]; then + info "$(bold)[Optional]$(regular) + To verify that deployment configuration for $(bold)${magento_ce_dir}$(regular) in PhpStorm is correct, + use instructions provided here: $(bold)https://github.com/paliarush/magento2-vagrant-for-developers/blob/2.0/docs/phpstorm-configuration-windows-hosts.md$(regular). + If not using PhpStorm, you can set up synchronization using rsync" +fi -bash "${vagrant_dir}/scripts/host/configure_php_storm.sh" +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/log/email/.gitignore b/log/email/.gitignore new file mode 100644 index 00000000000..f59ec20aabf --- /dev/null +++ b/log/email/.gitignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/m-bin-magento b/m-bin-magento new file mode 100755 index 00000000000..3d95fc2dc37 --- /dev/null +++ b/m-bin-magento @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +arguments=$@ +cd "${vagrant_dir}" +vagrant ssh -c "\$MAGENTO_ROOT/bin/magento $arguments" 2> >(logError) +# To debug, comment out line above and uncomment line below +# vagrant ssh -c "php -d xdebug.remote_autostart=1 \$MAGENTO_ROOT/bin/magento $arguments" 2> >(logError) + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-clear-cache b/m-clear-cache index 399147f2642..18549cd6ed1 100755 --- a/m-clear-cache +++ b/m-clear-cache @@ -1,9 +1,35 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")"; pwd) +set -e -# Enable trace printing and exit on the first error -set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD -cd ${vagrant_dir} -vagrant ssh -c "m-clear-cache" \ No newline at end of file +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +warm_up_cache=0 +while getopts 'w' flag; do + case "${flag}" in + w) warm_up_cache=1 ;; + *) error "Unexpected option" && exit 1;; + esac +done + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +bash "${vagrant_dir}/scripts/host/m_clear_cache.sh" "$@" 2> >(logError) + +if [[ ${warm_up_cache} -eq 1 ]]; then + incrementNestingLevel + bash "${vagrant_dir}/scripts/host/warm_up_cache.sh" 2> >(logError) + decrementNestingLevel +else + info "Use 'bash ./m-clear-cache -w' to get automatic cache warm up" +fi + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-composer b/m-composer index 78d81c36d35..34251a58e14 100755 --- a/m-composer +++ b/m-composer @@ -1,10 +1,19 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")"; pwd) -magento_ce_dir="${vagrant_dir}/magento2ce" +set -e -# Enable trace printing and exit on the first error -set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD -cd ${magento_ce_dir} -bash "${vagrant_dir}/scripts/host/composer.sh" "$@" \ No newline at end of file +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +bash "${vagrant_dir}/scripts/host/m_composer.sh" "$@" 2> >(logError) + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-reinstall b/m-reinstall index 1ed9b9a6808..b5a4c7544f3 100755 --- a/m-reinstall +++ b/m-reinstall @@ -1,9 +1,19 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")"; pwd) +set -e -# Enable trace printing and exit on the first error -set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD -cd ${vagrant_dir} -vagrant ssh -c "m-reinstall" \ No newline at end of file +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +bash "${vagrant_dir}/scripts/host/m_reinstall.sh" "$@" 2> >(logError) + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-search-engine b/m-search-engine new file mode 100755 index 00000000000..58d955d63dd --- /dev/null +++ b/m-search-engine @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +cd "${vagrant_dir}" +vagrant ssh -c "bash /vagrant/scripts/guest/m-search-engine ${@}" 2> >(logError) + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-switch-to-ce b/m-switch-to-ce index d6896ef4928..9aeb766878a 100755 --- a/m-switch-to-ce +++ b/m-switch-to-ce @@ -1,40 +1,19 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")"; pwd) -magento_ce_dir="${vagrant_dir}/magento2ce" -magento_ee_dir="${magento_ce_dir}/magento2ee" -host_os=$(bash "${vagrant_dir}/scripts/host/get_host_os.sh") -php_executable=$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh") +set -e -# Enable trace printing and exit on the first error -set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD -if [ ! -f ${magento_ee_dir}/app/etc/aliases_to_classes_map.json ]; then - echo "EE codebase is not available" - exit 0 -fi +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} -if [ ! -f ${magento_ce_dir}/app/etc/aliases_to_classes_map.json ]; then - echo "Already switched to CE" - exit 0 +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x fi -${php_executable} -f ${magento_ee_dir}/dev/tools/build-ee.php -- --command=unlink --ee-source="${magento_ee_dir}" --ce-source="${magento_ce_dir}" - -cd ${magento_ce_dir} -git checkout composer.json -git checkout composer.lock - -if [[ ${host_os} == "Windows" || $(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs") == 0 ]]; then - # Prevent issues on Windows with incorrect symlinks to files - if [ -f ${magento_ce_dir}/app/etc/aliases_to_classes_map.json ]; then - rm ${magento_ce_dir}/app/etc/aliases_to_classes_map.json - fi - if [ -f ${magento_ce_dir}/app/etc/enterprise/di.xml ]; then - rm ${magento_ce_dir}/app/etc/enterprise/di.xml - fi -fi +bash "${vagrant_dir}/scripts/host/m_switch_to_ce.sh" "$@" 2> >(logError) -bash "${vagrant_dir}/m-clear-cache" -bash -c "${vagrant_dir}/m-composer install" -bash "${vagrant_dir}/m-reinstall" \ No newline at end of file +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-switch-to-ee b/m-switch-to-ee index 360321df19f..83e192d8130 100755 --- a/m-switch-to-ee +++ b/m-switch-to-ee @@ -1,41 +1,19 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")"; pwd) -magento_ce_dir="${vagrant_dir}/magento2ce" -magento_ee_dir="${magento_ce_dir}/magento2ee" -host_os=$(bash "${vagrant_dir}/scripts/host/get_host_os.sh") -php_executable=$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh") +set -e -# Enable trace printing and exit on the first error -set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD -if [ ! -f ${magento_ee_dir}/app/etc/aliases_to_classes_map.json ]; then - echo "EE codebase is not available" - exit 0 -fi +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} -if [ -f ${magento_ce_dir}/app/etc/aliases_to_classes_map.json ]; then - echo "Already switched to EE" - exit 0 +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x fi -${php_executable} -f ${magento_ee_dir}/dev/tools/build-ee.php -- --command=link --ee-source="${magento_ee_dir}" --ce-source="${magento_ce_dir}" - -cp ${magento_ee_dir}/composer.json ${magento_ce_dir}/composer.json -cp ${magento_ee_dir}/composer.lock ${magento_ce_dir}/composer.lock - -if [[ ${host_os} == "Windows" || $(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs") == 0 ]]; then - # Prevent issues on Windows with incorrect symlinks to files - if [ -f ${magento_ee_dir}/app/etc/aliases_to_classes_map.json ]; then - rm ${magento_ce_dir}/app/etc/aliases_to_classes_map.json - cp ${magento_ee_dir}/app/etc/aliases_to_classes_map.json ${magento_ce_dir}/app/etc/aliases_to_classes_map.json - fi - if [ -f ${magento_ee_dir}/app/etc/enterprise/di.xml ]; then - rm ${magento_ce_dir}/app/etc/enterprise/di.xml - cp ${magento_ee_dir}/app/etc/enterprise/di.xml ${magento_ce_dir}/app/etc/enterprise/di.xml - fi -fi +bash "${vagrant_dir}/scripts/host/m_switch_to_ee.sh" "$@" 2> >(logError) -bash "${vagrant_dir}/m-clear-cache" -bash -c "${vagrant_dir}/m-composer install" -bash "${vagrant_dir}/m-reinstall" \ No newline at end of file +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/m-varnish b/m-varnish new file mode 100755 index 00000000000..a2e2b797f1c --- /dev/null +++ b/m-varnish @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")" && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" +resetNestingLevel +current_script_name=`basename "$0"` +initLogFile ${current_script_name} + +debug_vagrant_project="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_vagrant_project")" +if [[ ${debug_vagrant_project} -eq 1 ]]; then + set -x +fi + +cd "${vagrant_dir}" +vagrant ssh -c "bash /vagrant/scripts/guest/m-varnish ${@}" 2> >(logError) + +info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)" diff --git a/scripts/colors.sh b/scripts/colors.sh new file mode 100644 index 00000000000..e0819b0f11c --- /dev/null +++ b/scripts/colors.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# CLI color functions +function red() +{ + printf "\e[31m" +} + +function bold() +{ + printf "\e[1m" +} + +function green() +{ + printf "\e[32m" +} + +function yellow() +{ + printf "\e[33m" +} + +function blue() +{ + printf "\e[34m" +} + +function grey() +{ + printf "\e[37m" +} + +function regular() +{ + printf "\e[m" +} diff --git a/scripts/get_config_value.sh b/scripts/get_config_value.sh index 2c59746db46..23d0ce0180c 100644 --- a/scripts/get_config_value.sh +++ b/scripts/get_config_value.sh @@ -3,9 +3,9 @@ parse_yaml() { # src: https://gist.github.com/pkuczynski/8665367 local prefix=$2 - local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs="$(echo @|tr @ '\034')" sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ - -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" "$1" | awk -F$fs '{ indent = length($1)/2; vname[indent] = $2; @@ -17,11 +17,14 @@ parse_yaml() { }' } -vagrant_dir=$(cd "$(dirname "$0")/.."; pwd) +cd "$(dirname "${BASH_SOURCE[0]}")/.." && vagrant_dir=$PWD + variable_name=$1 # Read configs eval $(parse_yaml "${vagrant_dir}/etc/config.yaml.dist") -eval $(parse_yaml "${vagrant_dir}/etc/config.yaml") +if [[ -f "${vagrant_dir}/etc/config.yaml" ]]; then + eval $(parse_yaml "${vagrant_dir}/etc/config.yaml") +fi -echo ${!variable_name} \ No newline at end of file +echo ${!variable_name} diff --git a/scripts/guest/change_magento_config_for_functional_tests b/scripts/guest/change_magento_config_for_functional_tests new file mode 100755 index 00000000000..0b68188d9c3 --- /dev/null +++ b/scripts/guest/change_magento_config_for_functional_tests @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Changing Magento configuration to allow execution of functional tests" +incrementNestingLevel + +update_magento_config 'cms/wysiwyg/enabled' 'disabled' +update_magento_config 'admin/security/admin_account_sharing' '1' +update_magento_config 'admin/security/use_form_key' '0' + +decrementNestingLevel diff --git a/scripts/guest/check_mounted_directories b/scripts/guest/check_mounted_directories new file mode 100755 index 00000000000..63f7473b0fd --- /dev/null +++ b/scripts/guest/check_mounted_directories @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Checking if all required directories were mounted successfully" +incrementNestingLevel + +if [[ ! -f "${MAGENTO_ROOT}/bin/magento" ]]; then + error "Directory '${MAGENTO_ROOT}' was not mounted as expected and Magento code base is not accessible on the guest machine. + If your host is OSX or *nix, please make sure that Vagrant is able to mount NFS shares on your environment (see https://github.com/paliarush/magento2-vagrant-for-developers/issues/88#issuecomment-254854019 ). + Also remove any stale declarations from /etc/exports on the host." + exit 1 +fi + +if [[ ! -f "/vagrant/etc/config.yaml" ]]; then + error "Directory '/vagrant/etc' was not mounted as expected by Vagrant and is not accessible on the guest machine. + Please make sure that Vagrant is able to mount VirtualBox shared folders on your environment (see https://www.vagrantup.com/docs/synced-folders/basic_usage.html ). + Also remove any stale declarations from /etc/exports on the host." + exit 1 +fi + +decrementNestingLevel diff --git a/scripts/guest/configure_cache_backend b/scripts/guest/configure_cache_backend new file mode 100755 index 00000000000..5f6459c6bbc --- /dev/null +++ b/scripts/guest/configure_cache_backend @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Configuring cache backend according to config.yaml" +incrementNestingLevel + +cache_backend="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_cache_backend")" +magento_composer_content="$(cat "${MAGENTO_ROOT}/composer.json")" +redis_configuration=" + 'cache' => [ + 'frontend' => [ + 'default' => [ + 'backend' => 'Cm_Cache_Backend_Redis', + 'backend_options' => [ + 'server' => '127.0.0.1', + 'port' => '6379' + ], + ], + 'page_cache' => [ + 'backend' => 'Cm_Cache_Backend_Redis', + 'backend_options' => [ + 'server' => '127.0.0.1', + 'port' => '6379', + 'database' => '1', + 'compress_data' => '0' + ] + ] + ] + ], +" + +# Removing existing configuration +perl -i -p0e "s/,\s*'cache'.*\],/,/smg" "${MAGENTO_ROOT}/app/etc/env.php" + +incompatible_magento_version_pattern='"version": "2.0.[0-5]' +if [[ ${cache_backend} == "redis" ]] && [[ ! ${magento_composer_content} =~ ${incompatible_magento_version_pattern} ]]; then + status "Using Redis backend for caching" + perl -i -p0e "s/\n*\);/${redis_configuration});/smg" "${MAGENTO_ROOT}/app/etc/env.php" + redis-cli flushall 2> >(logError) > >(log) +else + status "Using file system backend for caching" +fi + +decrementNestingLevel diff --git a/scripts/guest/configure_debugging b/scripts/guest/configure_debugging new file mode 100755 index 00000000000..8dfa97af792 --- /dev/null +++ b/scripts/guest/configure_debugging @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +debug_magento_storefront="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_magento_storefront")" +if [[ ${debug_magento_storefront} == 1 ]]; then + status "Enabling Magento storefront debugging (as specified in etc/config.yaml)" + incrementNestingLevel + update_magento_config 'dev/debug/template_hints_storefront' '1' + update_magento_config 'dev/debug/template_hints_blocks' '1' + decrementNestingLevel +else + update_magento_config 'dev/debug/template_hints_storefront' '0' 2> >(logError) > >(log) + update_magento_config 'dev/debug/template_hints_blocks' '0' 2> >(logError) > >(log) +fi + +debug_magento_admin="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "debug_magento_admin")" +if [[ ${debug_magento_admin} == 1 ]]; then + status "Enabling Magento admin debugging (as specified in etc/config.yaml)" + incrementNestingLevel + update_magento_config 'dev/debug/template_hints_admin' '1' + decrementNestingLevel +else + update_magento_config 'dev/debug/template_hints_admin' '0' 2> >(logError) > >(log) +fi + +# Switch to desired Magento mode +env_php_content="$(cat "${MAGENTO_ROOT}/app/etc/env.php")" +magento_mode="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_mode")" +status "Checking if Magento mode should be changed to '${magento_mode}' (as specified in etc/config.yaml)" +mage_mode_pattern="'MAGE_MODE' => '${magento_mode}" +if [[ ! ${env_php_content} =~ ${mage_mode_pattern} ]]; then + status "Enabling Magento '${magento_mode}' mode" + php "${MAGENTO_ROOT}/bin/magento" deploy:mode:set ${magento_mode} 2> >(logError) > >(log) +fi diff --git a/scripts/guest/configure_search_engine b/scripts/guest/configure_search_engine new file mode 100755 index 00000000000..dcac3986d7d --- /dev/null +++ b/scripts/guest/configure_search_engine @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Init environment variables +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Configuring search engine according to config.yaml" +incrementNestingLevel + +if [[ -z "$(grep "search_engine:" /vagrant/etc/config.yaml)" ]]; then + sed -i '/environment:/a \ \ search_engine: "mysql"' /vagrant/etc/config.yaml +fi + +if [[ -f "${MAGENTO_ROOT}/LICENSE_EE.txt" ]]; then + search_engine="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_search_engine")" + if [[ ${search_engine} == "elasticsearch" ]]; then + status "Configuring ElasticSearch search engine" + update_magento_config 'catalog/search/engine' 'elasticsearch' + update_magento_config 'catalog/search/elasticsearch_server_hostname' 'localhost' + update_magento_config 'catalog/search/elasticsearch_server_port' '9200' + update_magento_config 'catalog/search/elasticsearch_index_prefix' 'magento2' + update_magento_config 'catalog/search/elasticsearch_enable_auth' '0' + update_magento_config 'catalog/search/elasticsearch_server_timeout' '15' + elif [[ ${search_engine} == "mysql" ]]; then + status "Configuring MySQL search engine" + update_magento_config 'catalog/search/engine' 'mysql' + fi + + bash m-clear-cache + + status "Rebuilding catalog search index" + php "${MAGENTO_ROOT}/bin/magento" indexer:reindex catalogsearch_fulltext 2> >(logError) > >(log) +else + status "Search engine configuration was not changed (custom search engine is not available in Magento CE)." +fi + +decrementNestingLevel diff --git a/scripts/guest/configure_varnish b/scripts/guest/configure_varnish new file mode 100755 index 00000000000..1ae2f60abea --- /dev/null +++ b/scripts/guest/configure_varnish @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Configuring Varnish FPC according to config.yaml" +incrementNestingLevel + +# Get args +while getopts 'f' flag; do + case "${flag}" in + f) force_restart_services=1 ;; + *) error "Unexpected option" && decrementNestingLevel && exit 1;; + esac +done + +if [[ -z "$(grep "use_varnish:" /vagrant/etc/config.yaml)" ]]; then + sed -i '/environment:/a \ \ use_varnish: 0' /vagrant/etc/config.yaml +fi + +# Configure Varnish if enabled in config +restart_services=0 +use_varnish="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_use_varnish")" +if [[ $use_varnish == 1 ]]; then + status "Configuring apache files to be ready for varnish" + if [[ -z "$(grep "VirtualHost\s\*:8080" /etc/apache2/sites-enabled/magento2.conf)" ]]; then + sudo sed -ie "s//" /etc/apache2/sites-enabled/magento2.conf + restart_services=1 + fi + if [[ -z "$(grep "VirtualHost\s\*:8080" /etc/apache2/sites-available/magento2.conf)" ]]; then + sudo sed -ie "s//" /etc/apache2/sites-available/magento2.conf + restart_services=1 + fi + if [[ -z "$(grep "Listen\s8080" /etc/apache2/ports.conf)" ]]; then + sudo sed -ie "s/Listen 80/Listen 8080/" /etc/apache2/ports.conf + restart_services=1 + fi + + status "Updating Magento database to use varnish FPC" + mysql -D magento -e "INSERT INTO core_config_data + ( scope, scope_id, path, value ) VALUES + ( 'default', '0', 'system/full_page_cache/caching_application', '2' ) + ON DUPLICATE KEY UPDATE value = 2 + ;" +else + status "Configuring apache files to run without varnish" + if [[ -z "$(grep "VirtualHost\s\*:80\b" /etc/apache2/sites-enabled/magento2.conf)" ]]; then + sudo sed -ie "s//" /etc/apache2/sites-enabled/magento2.conf + restart_services=1 + fi + if [[ -z "$(grep "VirtualHost\s\*:80\b" /etc/apache2/sites-available/magento2.conf)" ]]; then + sudo sed -ie "s//" /etc/apache2/sites-available/magento2.conf + restart_services=1 + fi + if [[ -z "$(grep "Listen\s80\b" /etc/apache2/ports.conf)" ]]; then + sudo sed -ie "s/Listen 8080/Listen 80/" /etc/apache2/ports.conf + restart_services=1 + fi + + status "Updating Magento database to not use varnish FPC" + mysql -D magento -e "INSERT INTO core_config_data + ( scope, scope_id, path, value ) VALUES + ( 'default', '0', 'system/full_page_cache/caching_application', '1' ) + ON DUPLICATE KEY UPDATE value = 1 + ;" +fi + +# Check if need restart services +if [[ $force_restart_services == 1 ]]; then + restart_services=1 +fi +if [[ $restart_services == 1 ]]; then + status "Restarting Apache and Varnish" + if [[ "$(ps -ax | pgrep varnish)" ]]; then + sudo pkill varnishd + fi + sudo service apache2 restart 2> >(logError) > >(log) + if [[ $use_varnish == 1 ]]; then + sudo varnishd -f /etc/varnish/default.vcl + fi + bash m-clear-cache +fi + +decrementNestingLevel diff --git a/scripts/guest/generate_basic_data b/scripts/guest/generate_basic_data new file mode 100755 index 00000000000..5a75c78aa19 --- /dev/null +++ b/scripts/guest/generate_basic_data @@ -0,0 +1,615 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +host_name="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_host_name")" + +status "Generating sample data" + +incrementNestingLevel + +adminToken=$(curl -sb -X POST "http://${host_name}/rest/V1/integration/admin/token" \ + -H "Content-Type:application/json" \ + -d '{"username":"admin", "password":"123123q"}') +adminToken=$(echo ${adminToken} | sed -e 's/"//g') + +category_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/categories" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "category": { + "parentId": 2, + "name": "Test Category", + "isActive": true, + "includeInMenu": true + } + }') + +pattern='\{\"id\":([0-9]+),' +if [[ "${category_creation_response}" =~ ${pattern} ]]; then + category_id=${BASH_REMATCH[1]} +fi + +attribute_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products/attributes/color/options" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "option": { + "label": "red" + } + }') + +attribute_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products/attributes/color/options" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "option": { + "label": "blue" + } + }') + +attribute_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products/attributes/color/options" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "option": { + "label": "green" + } + }') + +attribute_set_add_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products/attribute-sets/attributes" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "attributeSetId": 4, + "attributeGroupId": 7, + "attributeCode": "color", + "sortOrder": 0 + }') + +product_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testSimpleProduct", + "name": "Test Simple Product", + "attribute_set_id": 4, + "price": 22, + "weight": 1, + "status": 1, + "visibility": 4, + "type_id": "simple", + "extension_attributes": { + "stock_item": { + "qty": 12345, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + } + ] + } + }') + +product_creation_response2=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testSimpleProduct2", + "name": "Test Simple Product 2", + "attribute_set_id": 4, + "price": 32, + "weight": 1, + "status": 1, + "visibility": 4, + "type_id": "simple", + "extension_attributes": { + "stock_item": { + "qty": 12345, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + } + ] + } + }') + +product_creation_response_child1=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testConfigProduct-red", + "name": "Test Red Configurable Product", + "attribute_set_id": 4, + "price": 10, + "weight": 1, + "status": 1, + "visibility": 1, + "type_id": "simple", + "extension_attributes": { + "stock_item": { + "qty": 1000, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + }, + { + "attribute_code": "color", + "value": "4" + } + ] + } + }') + +product_creation_response_child2=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testConfigProduct-blue", + "name": "Test Blue Configurable Product", + "attribute_set_id": 4, + "price": 10, + "weight": 1, + "status": 1, + "visibility": 1, + "type_id": "simple", + "extension_attributes": { + "stock_item": { + "qty": 1000, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + }, + { + "attribute_code": "color", + "value": "5" + } + ] + } + }') + +product_creation_response_child3=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testConfigProduct-green", + "name": "Test Green Configurable Product", + "attribute_set_id": 4, + "price": 10, + "weight": 1, + "status": 1, + "visibility": 1, + "type_id": "simple", + "extension_attributes": { + "stock_item": { + "qty": 1000, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + }, + { + "attribute_code": "color", + "value": "6" + } + ] + } + }') + +config_product_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "sku": "testConfigProduct", + "name": "Test Configurable Product", + "attribute_set_id": 4, + "price": 10, + "weight": 1, + "status": 1, + "visibility": 4, + "type_id": "configurable", + "extension_attributes": { + "stock_item": { + "qty": 1000, + "is_in_stock": true + } + }, + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "'"${category_id}"'" + ] + } + ] + } + }') + +option_creation_resp=$(curl -sb -X POST "http://${host_name}/rest/V1/configurable-products/testConfigProduct/options" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "option": { + "attribute_id": "93", + "label": "Color", + "position": 0, + "values": [ + { + "value_index": 4 + }, + { + "value_index": 5 + }, + { + "value_index": 6 + } + ] + } + }') + +product_creation_response_child1=$(curl -sb -X PUT "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "id": 3, + "sku": "testConfigProduct-red", + "name": "Test Red Configurable Product", + "attribute_set_id": 4, + "price": 10, + "status": 1, + "visibility": 1, + "type_id": "simple", + "weight": 1, + "extension_attributes": { + "website_ids": [ + 1 + ], + "category_links": [ + { + "position": 0, + "category_id": "3" + } + ], + "stock_item": { + "item_id": 3, + "product_id": 3, + "stock_id": 1, + "qty": 1000, + "is_in_stock": true, + "is_qty_decimal": false, + "show_default_notification_message": false, + "use_config_min_qty": true, + "min_qty": 0, + "use_config_min_sale_qty": 1, + "min_sale_qty": 1, + "use_config_max_sale_qty": true, + "max_sale_qty": 10000, + "use_config_backorders": true, + "backorders": 0, + "use_config_notify_stock_qty": true, + "notify_stock_qty": 1, + "use_config_qty_increments": true, + "qty_increments": 0, + "use_config_enable_qty_inc": true, + "enable_qty_increments": false, + "use_config_manage_stock": true, + "manage_stock": true, + "low_stock_date": null, + "is_decimal_divided": false, + "stock_status_changed_auto": 0 + } + }, + "product_links": [], + "options": [], + "media_gallery_entries": [], + "tier_prices": [], + "custom_attributes": [ + { + "attribute_code": "color", + "value": "4" + }, + { + "attribute_code": "category_ids", + "value": [ + "3" + ] + }, + { + "attribute_code": "options_container", + "value": "container2" + }, + { + "attribute_code": "required_options", + "value": "0" + }, + { + "attribute_code": "has_options", + "value": "0" + }, + { + "attribute_code": "url_key", + "value": "test-red-configurable-product" + }, + { + "attribute_code": "tax_class_id", + "value": "2" + } + ] + } + }') + +product_creation_response_child2=$(curl -sb -X PUT "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "id": 4, + "sku": "testConfigProduct-blue", + "name": "Test Blue Configurable Product", + "attribute_set_id": 4, + "price": 10, + "status": 1, + "visibility": 1, + "type_id": "simple", + "created_at": "2017-09-12 01:50:09", + "updated_at": "2017-09-12 01:50:09", + "weight": 1, + "extension_attributes": { + "website_ids": [ + 1 + ], + "category_links": [ + { + "position": 0, + "category_id": "3" + } + ], + "stock_item": { + "item_id": 4, + "product_id": 4, + "stock_id": 1, + "qty": 1000, + "is_in_stock": true, + "is_qty_decimal": false, + "show_default_notification_message": false, + "use_config_min_qty": true, + "min_qty": 0, + "use_config_min_sale_qty": 1, + "min_sale_qty": 1, + "use_config_max_sale_qty": true, + "max_sale_qty": 10000, + "use_config_backorders": true, + "backorders": 0, + "use_config_notify_stock_qty": true, + "notify_stock_qty": 1, + "use_config_qty_increments": true, + "qty_increments": 0, + "use_config_enable_qty_inc": true, + "enable_qty_increments": false, + "use_config_manage_stock": true, + "manage_stock": true, + "low_stock_date": null, + "is_decimal_divided": false, + "stock_status_changed_auto": 0 + } + }, + "product_links": [], + "options": [], + "media_gallery_entries": [], + "tier_prices": [], + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "3" + ] + }, + { + "attribute_code": "options_container", + "value": "container2" + }, + { + "attribute_code": "required_options", + "value": "0" + }, + { + "attribute_code": "color", + "value": "5" + }, + { + "attribute_code": "has_options", + "value": "0" + }, + { + "attribute_code": "url_key", + "value": "test-blue-configurable-product" + }, + { + "attribute_code": "tax_class_id", + "value": "2" + } + ] + } + }') + +product_creation_response_child3=$(curl -sb -X PUT "http://${host_name}/rest/V1/products" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "product": { + "id": 5, + "sku": "testConfigProduct-green", + "name": "Test Green Configurable Product", + "attribute_set_id": 4, + "price": 10, + "status": 1, + "visibility": 1, + "type_id": "simple", + "created_at": "2017-09-12 01:50:10", + "updated_at": "2017-09-12 01:50:10", + "weight": 1, + "extension_attributes": { + "website_ids": [ + 1 + ], + "category_links": [ + { + "position": 0, + "category_id": "3" + } + ], + "stock_item": { + "item_id": 5, + "product_id": 5, + "stock_id": 1, + "qty": 1000, + "is_in_stock": true, + "is_qty_decimal": false, + "show_default_notification_message": false, + "use_config_min_qty": true, + "min_qty": 0, + "use_config_min_sale_qty": 1, + "min_sale_qty": 1, + "use_config_max_sale_qty": true, + "max_sale_qty": 10000, + "use_config_backorders": true, + "backorders": 0, + "use_config_notify_stock_qty": true, + "notify_stock_qty": 1, + "use_config_qty_increments": true, + "qty_increments": 0, + "use_config_enable_qty_inc": true, + "enable_qty_increments": false, + "use_config_manage_stock": true, + "manage_stock": true, + "low_stock_date": null, + "is_decimal_divided": false, + "stock_status_changed_auto": 0 + } + }, + "product_links": [], + "options": [], + "media_gallery_entries": [], + "tier_prices": [], + "custom_attributes": [ + { + "attribute_code": "category_ids", + "value": [ + "3" + ] + }, + { + "attribute_code": "options_container", + "value": "container2" + }, + { + "attribute_code": "required_options", + "value": "0" + }, + { + "attribute_code": "color", + "value": "6" + }, + { + "attribute_code": "has_options", + "value": "0" + }, + { + "attribute_code": "url_key", + "value": "test-green-configurable-product" + }, + { + "attribute_code": "tax_class_id", + "value": "2" + } + ] + } + }') + +child_create_response1=$(curl -sb -X POST "http://${host_name}/rest/V1/configurable-products/testConfigProduct/child" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{"childSku": "testConfigProduct-red"}') +child_create_response2=$(curl -sb -X POST "http://${host_name}/rest/V1/configurable-products/testConfigProduct/child" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{"childSku": "testConfigProduct-blue"}') +child_create_response3=$(curl -sb -X POST "http://${host_name}/rest/V1/configurable-products/testConfigProduct/child" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{"childSku": "testConfigProduct-green"}') + +customer_creation_response=$(curl -sb -X POST "http://${host_name}/rest/V1/customers" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${adminToken}" \ + -d '{ + "customer": { + "group_id":1, + "email":"customer@example.com", + "firstname":"John", + "lastname":"Doe", + "store_id":1, + "addresses":[ + { + "region": { + "region_code": "TX", + "region": "Texas", + "region_id": 57 + }, + "region_id": 57, + "country_id": "US", + "street": [ + "7700 Parmer Lane" + ], + "telephone": "5555555555", + "postcode": "78729", + "city": "Austin", + "firstname": "John", + "lastname": "Doe", + "default_shipping": true, + "default_billing": true + } + ], + "disable_auto_group_change": 0 + }, + "password": "123123qQ" + }') + +decrementNestingLevel diff --git a/scripts/guest/link_configs b/scripts/guest/link_configs new file mode 100755 index 00000000000..9bac600e146 --- /dev/null +++ b/scripts/guest/link_configs @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +function process_configs () { + configs_path=$1 + configs=$2 + for config in "${configs[@]}" + do + if [[ ! -d /vagrant/etc/guest/${config} ]] && [[ ! -f /vagrant/etc/guest/${config} ]]; then + if [[ -d ${configs_path}/${config} ]] || [[ -f ${configs_path}/${config} ]]; then + sudo rm -rf "${configs_path}/${config}.back" + sudo cp -rp ${configs_path}/${config} "${configs_path}/${config}.back" + sudo mv ${configs_path}/${config} /vagrant/etc/guest/${config} + sudo ln -s /vagrant/etc/guest/${config} ${configs_path}/${config} + fi + fi + done +} + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +# Below configuration is required to allow managing mysql as a service +if ! cat /etc/apparmor.d/local/usr.sbin.mysqld | grep -q '/vagrant/etc/guest' ; then + echo " + /vagrant/etc/guest/mysql/*.pem r, + /vagrant/etc/guest/mysql/conf.d/ r, + /vagrant/etc/guest/mysql/conf.d/* r, + /vagrant/etc/guest/mysql/*.cnf r," >> /etc/apparmor.d/local/usr.sbin.mysqld +fi + +status "Making guest configs visible and editable in the host IDE" +incrementNestingLevel + +# Configs located under /etc/* +config_dir="/etc" +# See unlink_configs script +configs=( apache2 php mysql varnish rabbitmq ) +process_configs ${config_dir} ${configs} + +decrementNestingLevel diff --git a/scripts/guest/log_email b/scripts/guest/log_email new file mode 100755 index 00000000000..3e0e6a8367c --- /dev/null +++ b/scripts/guest/log_email @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +email_log_dir=$1 +if [[ ! -d ${email_log_dir} ]]; then + mkdir -p "${email_log_dir}" +fi + +# Construct email file path +current_time="$( date +"%Y-%m-%d_%H-%M-%S" )" +# Add random suffix to file name just in case there several emails sent in one second +random_suffix="$(( RANDOM % 100 ))" +email_file_path="${email_log_dir}/${current_time}_${random_suffix}" + +raw_email_output="$(cat)" + +# Add email topic to file name +pattern="Subject: (.*)\sX-PHP-Originating-Script:" +if [[ "${raw_email_output}" =~ ${pattern} ]]; then + email_topic=${BASH_REMATCH[1]} + email_file_path="${email_file_path}_${email_topic}" +fi +email_file_path="${email_file_path}.html" + +# Output content +echo ${raw_email_output} > "${email_file_path}" + +# Process raw content of an email to make it a valid HTML +sed -i "s|.*<\!DOCTYPE|<\!DOCTYPE|g" "${email_file_path}" +sed -i "s|=3D|=|g" "${email_file_path}" +sed -i "s|=0A||g" "${email_file_path}" +sed -i "s|= ||g" "${email_file_path}" \ No newline at end of file diff --git a/scripts/guest/m-clear-cache b/scripts/guest/m-clear-cache index a50f224a2a7..858821ea046 100755 --- a/scripts/guest/m-clear-cache +++ b/scripts/guest/m-clear-cache @@ -1,17 +1,73 @@ #!/usr/bin/env bash -echo "Clearing Magento cache..." - -rm -f ${MAGENTO_ROOT}/app/etc/paths.php -#clear var -for dir in `ls ${MAGENTO_ROOT}/var` ; do - rm -rf ${MAGENTO_ROOT}/var/${dir}/* ; -done -#clear pub/statics -cd ${MAGENTO_ROOT}/pub && mv static/.htaccess static_htaccess.back && rm -rf static/* && mv static_htaccess.back static/.htaccess -#clear integration tests tmp -cd ${MAGENTO_ROOT}/dev/tests/integration && mv tmp/.gitignore tmp_gitignore.back && rm -rf tmp/* && mv tmp_gitignore.back tmp/.gitignore -#clear unit tests tmp -cd ${MAGENTO_ROOT}/dev/tests/unit && rm -rf tmp/* - -echo "Magento cache cleared." +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +if [[ ${SKIP_CACHE_CLEAN} -eq 1 ]] ; then + exit 0 +fi + +status "Clearing Magento cache" +incrementNestingLevel + +status "Clearing directories containing temporary files" + +mage_mode=$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_mode") + +rm -f "${MAGENTO_ROOT}/app/etc/paths.php" +if [[ -d "${MAGENTO_ROOT}/var" ]]; then + # clear var + cd "${MAGENTO_ROOT}" + mv var/.htaccess var_htaccess.back + if [[ -d "${MAGENTO_ROOT}/var/generation" ]] && [[ ${mage_mode} == "production" ]]; then + # for Magento v2.0.x and v2.1.x + mv var/generation var_generation.back + fi + rm -rf var/* var/.[^.]* + if [[ -f var_htaccess.back ]]; then + mv var_htaccess.back var/.htaccess + fi + if [[ -d "${MAGENTO_ROOT}/var_generation.back" ]] && [[ ${mage_mode} == "production" ]]; then + mv var_generation.back var/generation + fi +fi + +if [[ -d "${MAGENTO_ROOT}/generated" ]] && [[ ${mage_mode} != "production" ]]; then + # for Magento v2.2.0 and greater + cd "${MAGENTO_ROOT}" + rm -rf generated/* +fi + +if [[ -d "${MAGENTO_ROOT}/pub" ]] && [[ ${mage_mode} != "production" ]]; then + # clear pub/statics + cd "${MAGENTO_ROOT}/pub" && mv static/.htaccess static_htaccess.back && rm -rf static && mkdir static + if [[ -f static_htaccess.back ]]; then + mv static_htaccess.back static/.htaccess + fi +fi + +if [[ -d "${MAGENTO_ROOT}/dev" ]]; then + # clear integration tests tmp + cd "${MAGENTO_ROOT}/dev/tests/integration" && mv tmp/.gitignore tmp_gitignore.back && rm -rf tmp && mkdir tmp + if [[ -f tmp_gitignore.back ]]; then + mv tmp_gitignore.back tmp/.gitignore + fi + # clear unit tests tmp + cd "${MAGENTO_ROOT}/dev/tests/unit" && mv tmp/.gitignore tmp_gitignore.back && rm -rf tmp && mkdir tmp + if [[ -f tmp_gitignore.back ]]; then + mv tmp_gitignore.back tmp/.gitignore + fi +fi + +if [[ -f "${MAGENTO_ROOT}/app/etc/config.php" ]]; then + + bash configure_debugging + + bash configure_cache_backend + + status "Flushing cache using 'bin/magento cache:flush'" + php "${MAGENTO_ROOT}/bin/magento" cache:flush 2> >(logError) > >(log) +fi + +decrementNestingLevel diff --git a/scripts/guest/m-reinstall b/scripts/guest/m-reinstall index e8a3f7b1f93..8b97fa866c0 100755 --- a/scripts/guest/m-reinstall +++ b/scripts/guest/m-reinstall @@ -1,19 +1,22 @@ #!/usr/bin/env bash -# Enable trace printing and exit on the first error -set -ex +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" get_config_value="/vagrant/scripts/get_config_value.sh" -use_nfs_for_synced_folders=$(bash ${get_config_value} "guest_use_nfs") -magento_host_name=$(bash ${get_config_value} "magento_host_name") +magento_host_name="$(bash ${get_config_value} "magento_host_name")" +use_nfs="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs")" +is_windows_host=${IS_WINDOWS_HOST} +generate_basic_data="$(bash ${get_config_value} "magento_generate_basic_data")" declare -A setupOptions -setupOptions[admin_frontname]=$(bash ${get_config_value} "magento_admin_frontname") -setupOptions[language]=$(bash ${get_config_value} "magento_language") -setupOptions[timezone]=$(bash ${get_config_value} "magento_timezone") -setupOptions[currency]=$(bash ${get_config_value} "magento_currency") -setupOptions[admin_user]=$(bash ${get_config_value} "magento_admin_user") -setupOptions[admin_password]=$(bash ${get_config_value} "magento_admin_password") +setupOptions[admin_frontname]="$(bash ${get_config_value} "magento_admin_frontname")" +setupOptions[language]="$(bash ${get_config_value} "magento_language")" +setupOptions[timezone]="$(bash ${get_config_value} "magento_timezone")" +setupOptions[currency]="$(bash ${get_config_value} "magento_currency")" +setupOptions[admin_user]="$(bash ${get_config_value} "magento_admin_user")" +setupOptions[admin_password]="$(bash ${get_config_value} "magento_admin_password")" setupOptions[db_host]='localhost' setupOptions[db_name]='magento' setupOptions[db_user]='root' @@ -25,19 +28,31 @@ setupOptions[amqp_host]='localhost' setupOptions[amqp_port]='5672' setupOptions[amqp_user]='guest' setupOptions[amqp_password]='guest' +setupOptions[amqp_virtualhost]='/' -cd ${MAGENTO_ROOT} +if [[ ${is_windows_host} == 1 ]] || [[ ${use_nfs} == 0 ]]; then + sudo chown -R vagrant:vagrant ${MAGENTO_ROOT} +fi + +status "Killing MQ processes" +pkill -f queue -# Clear cache -m-clear-cache +status "Installing/re-installing Magento" +incrementNestingLevel -# Remove configuration files +cd ${MAGENTO_ROOT} + +status "Removing Magento configuration files (env.php and config.php)" rm -f "${MAGENTO_ROOT}/app/etc/config.php" rm -f "${MAGENTO_ROOT}/app/etc/env.php" -# Create DB -db_names=(${setupOptions[db_name]} "magento_integration_tests" ) +bash m-clear-cache +# Cache cleaning takes 5-10 seconds and should be avoided until the end of installation to speed up the process +export SKIP_CACHE_CLEAN=1 + +db_names=("${setupOptions[db_name]}" "magento_integration_tests" ) for db_name in "${db_names[@]}"; do + status "Dropping and creating '${db_name}' DB" mysql -e "drop database if exists ${db_name}; create database ${db_name};" done @@ -62,40 +77,66 @@ install_cmd="./bin/magento setup:install \ --use-rewrites=1" # Configure Rabbit MQ -if [ -f "${MAGENTO_ROOT}/app/code/Magento/Amqp/registration.php" ]; then +if [[ -d "${MAGENTO_ROOT}/app/code/Magento/MessageQueue" ]] || [[ -d "${MAGENTO_ROOT}/vendor/magento/magento-message-queue" ]]; then install_cmd="${install_cmd} \ --amqp-host=${setupOptions[amqp_host]} \ --amqp-port=${setupOptions[amqp_port]} \ --amqp-user=${setupOptions[amqp_user]} \ + --amqp-virtualhost=${setupOptions[amqp_virtualhost]} \ --amqp-password=${setupOptions[amqp_password]}" fi -chmod +x bin/magento -php ${install_cmd} +sudo chmod +x bin/magento +status "${install_cmd}" +php ${install_cmd} 2> >(logError) > >(log) +# Comment out the line above and uncomment the one below to debug Magento Setup script +# php -d xdebug.remote_autostart=1 ${install_cmd} 2> >(logError) > >(log) -# Enable Magento cron jobs -echo "* * * * * php ${MAGENTO_ROOT}/bin/magento cron:run &" | crontab -u vagrant - +if [[ $? != 0 ]]; then + error "Magento installation failed." + exit 1 +fi + +bash "${vagrant_dir}/scripts/guest/configure_varnish" -f -if [ ${use_nfs_for_synced_folders} -eq 0 ]; then - chown -R vagrant:vagrant ${MAGENTO_ROOT} +status "Enabling Magento cron jobs" +echo "* * * * * php ${MAGENTO_ROOT}/bin/magento cron:run & +* * * * * php ${MAGENTO_ROOT}/update/cron.php & +* * * * * php ${MAGENTO_ROOT}/bin/magento setup:cron:run &" | crontab -u vagrant - + +if [[ ${is_windows_host} == 1 ]] || [[ ${use_nfs} == 0 ]]; then + status "Changing ownership of "${MAGENTO_ROOT}" to vagrant:vagrant" + sudo chown -R vagrant:vagrant ${MAGENTO_ROOT} fi -# Generate XSD references for PHP Storm -php bin/magento dev:urn-catalog:generate /vagrant/.idea/misc.xml -sed -i "s|${MAGENTO_ROOT}|${MAGENTO_ROOT_HOST}|g" "/vagrant/.idea/misc.xml" +if [[ ${generate_basic_data} != 0 ]]; then + bash generate_basic_data +fi +bash configure_search_engine +bash change_magento_config_for_functional_tests +bash update_magento_config 'admin/security/session_lifetime' '86400' + +export SKIP_CACHE_CLEAN=0 +bash m-clear-cache + +status "Checking if Magento frontend is accessible at '${setupOptions[base_url]}'" +magento_home_page_content="$(curl -sL ${setupOptions[base_url]})" +pattern="All rights reserved." +if [[ ${magento_home_page_content} =~ ${pattern} ]]; then + + bash warm_up_cache -set +x -echo " -Magento application was deployed to ${MAGENTO_ROOT} and installed successfully -Access storefront at ${setupOptions[base_url]} -Access admin panel at ${setupOptions[base_url]}${setupOptions[admin_frontname]}/" + status "Generating XSD references for PHP Storm" + php bin/magento dev:urn-catalog:generate /vagrant/.idea/misc.xml + sed -i "s|${MAGENTO_ROOT}|${MAGENTO_ROOT_HOST}|g" "/vagrant/.idea/misc.xml" -if [ ${use_nfs_for_synced_folders} -eq 0 ]; then - echo " - [Optional] To finish developer environment set up: - 1. Please create new PhpStorm project using 'magento2ce' directory on your host - (this directory should already contain Magento repository cloned earlier) + decrementNestingLevel + success "Magento reinstalled successfully" + info "Magento application was deployed to $(bold)${MAGENTO_ROOT}$(regular) and installed successfully - 2. Use instructions provided here https://github.com/paliarush/vagrant-magento/blob/2.0/docs/phpstorm-configuration-windows-hosts.md - to set up synchronization in PhpStorm (or using rsync) with ${MAGENTO_ROOT} directory" + Access storefront at $(bold)${setupOptions[base_url]}$(regular) + Access admin panel at $(bold)${setupOptions[base_url]}${setupOptions[admin_frontname]}/$(regular)" +else + error "Magento frontend is not accessible at '${setupOptions[base_url]}' after installation. Please scan output above and logs for errors" + exit 1 fi diff --git a/scripts/guest/m-search-engine b/scripts/guest/m-search-engine new file mode 100755 index 00000000000..c0fd5c7b1ab --- /dev/null +++ b/scripts/guest/m-search-engine @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Configuring search engine" +incrementNestingLevel + +if [[ $1 == "help" ]]; then + status "Usage: ./m-search-engine mysql/elasticsearch" + decrementNestingLevel + exit 0 +fi + + +# Check if user created config.yaml file +if [[ ! -f /vagrant/etc/config.yaml ]]; then + error "Please make sure you have create a config.yaml file copy from etc/config.yaml.dist" + decrementNestingLevel + exit 1 +fi + +if [[ -z "$(grep "search_engine:" /vagrant/etc/config.yaml)" ]]; then + sed -i '/environment:/a \ \ search_engine: "mysql"' /vagrant/etc/config.yaml +fi + +if [[ $1 == "mysql" ]]; then + status "Enabling MySQL search engine" + sed -ie 's/search_engine:.*/search_engine: "mysql"/' /vagrant/etc/config.yaml +elif [[ $1 == "elasticsearch" ]]; then + status "Enabling ElasticSearch search engine" + sed -ie 's/search_engine:.*/search_engine: "elasticsearch"/' /vagrant/etc/config.yaml +else + error "Usage: ./m-search-engine mysql|elasticsearch" + decrementNestingLevel + exit 1 +fi + +bash configure_search_engine + +decrementNestingLevel diff --git a/scripts/guest/m-varnish b/scripts/guest/m-varnish new file mode 100755 index 00000000000..5880adbb1da --- /dev/null +++ b/scripts/guest/m-varnish @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Configuring Varnish" +incrementNestingLevel + +if [[ $1 == "help" ]]; then + status "Usage: ./m-varnish enable/disable" + decrementNestingLevel + exit 0 +fi + +# Check if user created config.yaml file +if [[ ! -f /vagrant/etc/config.yaml ]]; then + error "Please make sure you have create a config.yaml file copy from etc/config.yaml.dist" + decrementNestingLevel + exit 1 +fi + +if [[ -z "$(grep "use_varnish:" /vagrant/etc/config.yaml)" ]]; then + sed -i '/environment:/a \ \ use_varnish: 0' /vagrant/etc/config.yaml +fi + +if [[ $1 == "enable" ]]; then + status "Enabling Varnish" + sed -ie "s/use_varnish:.*/use_varnish: 1/" /vagrant/etc/config.yaml +elif [[ $1 == "disable" ]]; then + status "Disabling Varnish" + sed -ie "s/use_varnish:.*/use_varnish: 0/" /vagrant/etc/config.yaml +else + error "Usage: ./m-varnish enable|disable" + decrementNestingLevel + exit 1 +fi + +bash configure_varnish + +decrementNestingLevel diff --git a/scripts/guest/unlink_configs b/scripts/guest/unlink_configs new file mode 100755 index 00000000000..f78b3a911b3 --- /dev/null +++ b/scripts/guest/unlink_configs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +function process_configs () { + configs_path=$1 + configs=$2 + for config in "${configs[@]}" + do + if [[ -L ${config_dir}/${config} ]] && [[ -e ${config_dir}/${config} ]]; then + sudo rm ${config_dir}/${config} + sudo mv /vagrant/etc/guest/${config} ${config_dir}/${config} + elif [[ ! -e ${config_dir}/${config} ]] && [[ -e "${config_dir}/${config}.back" ]]; then + sudo rm -rf ${config_dir}/${config} + sudo cp -rp "${config_dir}/${config}.back" ${config_dir}/${config} + fi + done +} + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Reverting configs before shutdown or reboot to allow proper services initialization on booting" +incrementNestingLevel + +# Configs located under /etc/* +config_dir="/etc" +# See link_configs script +configs=( apache2 php mysql varnish rabbitmq ) +process_configs ${config_dir} ${configs} + +decrementNestingLevel diff --git a/scripts/guest/update_magento_config b/scripts/guest/update_magento_config new file mode 100755 index 00000000000..103001b03f4 --- /dev/null +++ b/scripts/guest/update_magento_config @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +vagrant_dir="/vagrant" + +source "${vagrant_dir}/scripts/output_functions.sh" + +config_path=$1 +config_value=$2 + +status "Updating Magento config: ${config_path} = ${config_value}" +incrementNestingLevel + +mysql -D magento -e "INSERT INTO core_config_data + ( scope, scope_id, path, value ) VALUES + ( 'default', '0', '${config_path}', '${config_value}' ) + ON DUPLICATE KEY UPDATE value = '${config_value}' +;" + +decrementNestingLevel diff --git a/scripts/guest/warm_up_cache b/scripts/guest/warm_up_cache new file mode 100755 index 00000000000..aa8923a4de3 --- /dev/null +++ b/scripts/guest/warm_up_cache @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" + +function loadByUrl() +{ + log "Warming up ${1}" + wget -q -O - "${1}" | grep -o "http://${host_name}[^\"]*" | grep -o ".*\.[A-Za-z]*$" | while read -r link ; do + log "Warming up ${link}" + wget -q ${link} 2> >(log) > >(log) + done +} + +admin_frontname="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_admin_frontname")" +host_name="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_host_name")" + +status "Warming up Magento cache by loading frontend and admin pages" + +incrementNestingLevel + +loadByUrl "http://${host_name}" +loadByUrl "http://${host_name}/${admin_frontname}" + +decrementNestingLevel diff --git a/scripts/host/check_mounted_directories.sh b/scripts/host/check_mounted_directories.sh new file mode 100755 index 00000000000..e6540b319d6 --- /dev/null +++ b/scripts/host/check_mounted_directories.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" + +cd "${vagrant_dir}" +if [[ ! -f "${vagrant_dir}/etc/guest/mysql/my.cnf" ]]; then + error "Directory '${vagrant_dir}/etc' was not mounted as expected by Vagrant. + Please make sure that 'paliarush/magento2.ubuntu' Vagrant box was downloaded successfully (if not, this may help http://stackoverflow.com/questions/35519389/vagrant-cannot-find-box) + And that Vagrant is able to mount VirtualBox shared folders on your environment (see https://www.vagrantup.com/docs/synced-folders/basic_usage.html ). + Also remove any stale declarations from /etc/exports on the host." + exit 1 +fi +vagrant ssh -c "bash /vagrant/scripts/guest/check_mounted_directories" 2> >(logError) +# Explicit exit is necessary to bypass incorrect output from vagrant in case of errors +exit 0 diff --git a/scripts/host/check_requirements.sh b/scripts/host/check_requirements.sh index b32f91c6870..76ee3bc4401 100755 --- a/scripts/host/check_requirements.sh +++ b/scripts/host/check_requirements.sh @@ -1,19 +1,16 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")/../.."; pwd) -composer_auth_json="${vagrant_dir}/etc/composer/auth.json" -php_executable=$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh") +cd "$(dirname "${BASH_SOURCE[0]}")/../.." && vagrant_dir=$PWD -# Enable trace printing and exit on the first error -set -ex +source "${vagrant_dir}/scripts/output_functions.sh" -if ! ${php_executable} -v | grep -q 'Copyright' ; then +status "Checking requirements" +incrementNestingLevel + +php_executable="$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh")" + +if ! ${php_executable} -v 2> >(log) | grep -q 'Copyright' ; then bash "${vagrant_dir}/scripts/host/install_php.sh" fi -if [ ! -f ${composer_auth_json} ]; then - set +x - echo "Please specify GitHub token in ${composer_auth_json} to bypass GitHub rate limits (see https://github.com/paliarush/magento2-vagrant-for-developers/tree/develop#github-limitations)" - exit 255 - set -x -fi \ No newline at end of file +decrementNestingLevel diff --git a/scripts/host/composer.sh b/scripts/host/composer.sh index aa43946de5f..94312742e52 100755 --- a/scripts/host/composer.sh +++ b/scripts/host/composer.sh @@ -3,33 +3,44 @@ # This script allows to use credentials specified in etc/composer/auth.json without declaring them globally current_dir=${PWD} -vagrant_dir=$(cd "$(dirname "$0")/../.."; pwd) +cd "$(dirname "${BASH_SOURCE[0]}")/../.." && vagrant_dir=$PWD + +source "${vagrant_dir}/scripts/output_functions.sh" + +status "Executing composer command" +incrementNestingLevel + composer_auth_json="${vagrant_dir}/etc/composer/auth.json" composer_dir="${vagrant_dir}/scripts/host" composer_phar="${composer_dir}/composer.phar" -# Enable trace printing and exit on the first error -set -ex - bash "${vagrant_dir}/scripts/host/check_requirements.sh" -php_executable=$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh") +php_executable="$(bash "${vagrant_dir}/scripts/host/get_path_to_php.sh")" -# Setup composer if necessary -if [ ! -f ${composer_phar} ]; then - cd ${composer_dir} - curl -sS https://getcomposer.org/installer | ${php_executable} +if [[ ! -f ${composer_phar} ]]; then + status "Installing composer" + cd "${composer_dir}" + curl -sS https://getcomposer.org/installer | ${php_executable} 2> >(logError) > >(log) fi -# Configure composer credentials -cd ${current_dir} -cp ${composer_auth_json} "${PWD}/auth.json" +# commented out due to composer conflicts +# ${php_executable} "${composer_phar}" global require "hirak/prestissimo:^0.3" -host_os=$(bash "${vagrant_dir}/scripts/host/get_host_os.sh") +cd "${current_dir}" +if [[ -f ${composer_auth_json} ]]; then + status "Exporting etc/auth.json to environment variable" + export COMPOSER_AUTH="$(cat "${composer_auth_json}")" +fi + +host_os="$(bash "${vagrant_dir}/scripts/host/get_host_os.sh")" if [[ $(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_composer_prefer_source") == 1 ]]; then # prefer-source is slow but guarantees that there will be no issues related to max path length on Windows - ${php_executable} ${composer_phar} --ignore-platform-reqs --prefer-source "$@" + status "composer --ignore-platform-reqs --prefer-source --no-interaction "$@"" + "${php_executable}" "${composer_phar}" --ignore-platform-reqs --prefer-source --no-interaction "$@" 2> >(log) > >(log) else - ${php_executable} ${composer_phar} --ignore-platform-reqs "$@" + status "composer --ignore-platform-reqs --no-interaction "$@"" + "${php_executable}" "${composer_phar}" --ignore-platform-reqs --no-interaction "$@" 2> >(log) > >(log) fi -rm "${PWD}/auth.json" + +decrementNestingLevel diff --git a/scripts/host/configure_php_storm.sh b/scripts/host/configure_php_storm.sh index 747d8385226..36a46ee8009 100755 --- a/scripts/host/configure_php_storm.sh +++ b/scripts/host/configure_php_storm.sh @@ -1,21 +1,22 @@ #!/usr/bin/env bash -vagrant_dir=$(cd "$(dirname "$0")/../.."; pwd) -composer_auth_json="${vagrant_dir}/etc/composer/auth.json" +cd "$(dirname "${BASH_SOURCE[0]}")/../.." && vagrant_dir=$PWD -# Enable trace printing and exit on the first error -set +x +source "${vagrant_dir}/scripts/output_functions.sh" -cd ${vagrant_dir} -ssh_port=$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_forwarded_ssh_port") -magento_host_name=$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_host_name") +status "Configuring PhpStorm" +incrementNestingLevel + +cd "${vagrant_dir}" +ssh_port="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_forwarded_ssh_port")" +magento_host_name="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "magento_host_name")" cp -R "${vagrant_dir}/scripts/host/php-storm-configs/." "${vagrant_dir}/.idea/" enabled_virtual_host_config="/etc/apache2/sites-available/magento2.conf" -host_os=$(bash "${vagrant_dir}/scripts/host/get_host_os.sh") -if [[ ${host_os} == "Windows" || $(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs") == 0 ]]; then +host_os="$(bash "${vagrant_dir}/scripts/host/get_host_os.sh")" +if [[ ${host_os} == "Windows" ]] || [[ $(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs") == 0 ]]; then sed -i.back "s||/var/www/magento2ce|g" "${vagrant_dir}/.idea/deployment.xml" sed -i.back 's|| autoUpload="Always" autoUploadExternalChanges="true"|g' "${vagrant_dir}/.idea/deployment.xml" sed -i.back 's||