Merge branch 'better-process-handling'

CHANGELOG.md

@@ -1,6 +1,22 @@
 ### Changelog
 All notable changes to this project will be documented in this file.
 
+## [1.4.0] - 2019-02-26
+### Fixed
+- With multiple folders in the backup job the S3 paths were wrong
+### Changed
+- Now processes run in sequential order for better performance
+### Added
+- Stop backup button in job list view
+- Now the AWS CLI processes are killed on app exit
+- Now the current backup running process are killed after its done
+- Bucket info: size and number of objects are shown in S3 explorer view through AWS Cloudwatch
+- More app's icons for better rendering on Win 10 OS
+- Now you can limit S3 upload speed by adjust bandwidth and concurrent requests, 
+to avoid excessive band consumption
+- Now you can set the backup max duration time, in order to stop the backup after a certain 
+amount of time regardless the real time needed to complete the backup.
+
 ## [1.3.0] - 2019-02-26
 ### Added
 - Auto updater

README.md

@@ -26,7 +26,11 @@ You can find the AWS CLI installer here: [Download AWS CLI](https://aws.amazon.c
 
 ## Windows executable (portable and installer)
 
-[Check releases](https://github.com/ulver2812/aws-s3-backup/releases) 
+[Download here](https://github.com/ulver2812/aws-s3-backup/releases) 
+
+## Changelog
+
+[Check changelog](https://github.com/ulver2812/aws-s3-backup/blob/master/CHANGELOG.md) 
 
 ## Getting Started
 
@@ -75,7 +79,7 @@ Don't forget to deactivate the "Developer Tools" by commenting `win.webContents.
 ## AWS app settings
 
 In order to use the app you need to set in the settings page an "AWS access key ID" and an "AWS secret access key" that you can create through the IAM service in the AWS console. 
-The IAM user needs a programmatic access account with a correct read/write S3 policy attached. You can use any IAM S3 policy that grant access to the buckets that you want to use with the app.
+The IAM user needs a programmatic access account with a correct read/write S3 policy attached (e.g AmazonS3FullAccess) and CloudWatch Metrics (e.g CloudWatchReadOnlyAccess). You can use any IAM S3 policy that grant access to the buckets that you want to use with the app.
 Here an example policy: [IAM S3 example policy](https://docs.aws.amazon.com/en_us/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html)
 
 ## To correctly quit the app

main.ts

@@ -1,11 +1,24 @@
-import {app, BrowserWindow, BrowserWindowConstructorOptions, screen, Tray, Menu, nativeImage} from 'electron';
+import {
+  app,
+  BrowserWindow,
+  BrowserWindowConstructorOptions,
+  screen,
+  Tray,
+  Menu,
+  nativeImage,
+  ipcMain
+} from 'electron';
 import * as path from 'path';
 import * as url from 'url';
 import * as Splashscreen from '@trodi/electron-splashscreen';
 import * as contextMenuInternal from 'electron-context-menu';
+
 const {autoUpdater} = require('electron-updater');
+const sugar = require('sugar');
+const kill = require('tree-kill');
 
 let win, serve, tray;
+const awsCliProcesses = [];
 const args = process.argv.slice(1);
 serve = args.some(val => val === '--serve');
 
@@ -71,7 +84,7 @@ function createWindow() {
 }
 
 function createTray() {
-  const trayIcon = path.join(__dirname, 'icons/favicon.png');
+  const trayIcon = path.join(__dirname, 'icons/favicon.16x16.png');
   const nimage = nativeImage.createFromPath(trayIcon);
   tray = new Tray(nimage);
   const contextMenu = Menu.buildFromTemplate([
@@ -100,6 +113,16 @@ function checkForUpdate() {
   autoUpdater.checkForUpdatesAndNotify();
 }
 
+function initIpc() {
+  ipcMain.on('add-process-to-kill', (event, processPid) => {
+    awsCliProcesses.push(processPid);
+  });
+
+  ipcMain.on('remove-process-to-kill', (event, processPid) => {
+    sugar.Array.remove(awsCliProcesses, processPid);
+  });
+}
+
 try {
 
   // This method will be called when Electron has finished
@@ -113,6 +136,8 @@ try {
 
   app.on('ready', checkForUpdate);
 
+  app.on('ready', initIpc);
+
   // Quit when all windows are closed.
   app.on('window-all-closed', () => {
     // On OS X it is common for applications and their menu bar
@@ -130,6 +155,12 @@ try {
     }
   });
 
+  app.on('before-quit', function () {
+    for (const process of awsCliProcesses) {
+      kill(process);
+    }
+  });
+
 } catch (e) {
   // Catch Error
   // throw e;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aws-s3-backup",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "AWS S3 backup system",
   "homepage": "https://github.com/ulver2812/aws-s3-backup",
   "author": {
@@ -57,7 +57,8 @@
     "ngx-electron": "^1.0.4",
     "node-schedule": "^1.3.0",
     "nodemailer": "^5.1.1",
-    "sugar": "^2.0.4"
+    "sugar": "^2.0.4",
+    "tree-kill": "^1.2.1"
   },
   "devDependencies": {
     "@angular-devkit/build-angular": "0.6.3",

src/app/app.module.ts

@@ -37,6 +37,7 @@ import { JobAlertDialogComponent } from './components/dialogs/job-alert-dialog/j
 import { JobBackupManuallyComponent } from './components/dialogs/job-backup-manually/job-backup-manually.component';
 import { NoInternetConnectionComponent } from './components/dialogs/no-internet-connection/no-internet-connection.component';
 import {NotificationsService} from './providers/notifications.service';
+import {ProcessesHandlerService} from './providers/processes-handler.service';
 
 
 // AoT requires an exported function for factories
@@ -77,7 +78,7 @@ export function HttpLoaderFactory(http: HttpClient) {
     }),
     BrowserAnimationsModule
   ],
-  providers: [ElectronService, JobsService, JobSchedulerService, NotificationsService],
+  providers: [ElectronService, JobsService, JobSchedulerService, NotificationsService, ProcessesHandlerService],
   bootstrap: [AppComponent],
   entryComponents: [JobAlertDialogComponent, JobBackupManuallyComponent, NoInternetConnectionComponent]
 })

src/app/components/dialogs/job-backup-manually/job-backup-manually.component.ts

@@ -2,6 +2,8 @@ import {Component, Inject, OnInit} from '@angular/core';
 import {AwsService} from '../../../providers/aws.service';
 import {Job} from '../../../models/job.model';
 import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material';
+import {LogType} from '../../../enum/log.type.enum';
+import {LogService} from '../../../providers/log.service';
 
 @Component({
   selector: 'app-job-backup-manually',
@@ -10,9 +12,11 @@ import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material';
 })
 export class JobBackupManuallyComponent implements OnInit {
 
-  constructor(public dialogRef: MatDialogRef<JobBackupManuallyComponent>,
-              @Inject(MAT_DIALOG_DATA) public data: { job: Job },
-              private aws: AwsService
+  constructor(
+    public dialogRef: MatDialogRef<JobBackupManuallyComponent>,
+    @Inject(MAT_DIALOG_DATA) public data: { job: Job },
+    private aws: AwsService,
+    private logService: LogService
   ) {
     dialogRef.disableClose = true;
   }
@@ -21,6 +25,7 @@ export class JobBackupManuallyComponent implements OnInit {
   }
 
   startBackup() {
+    this.logService.printLog(LogType.INFO, 'The job ' + this.data.job.name + ' was started manually.');
     this.aws.s3Sync(this.data.job);
     this.dialogRef.close();
   }

src/app/components/edit-job/edit-job.component.html

@@ -88,6 +88,13 @@ <h3 class="mat-error">{{ 'PAGES.EDIT-JOB.FIELDS.SCHEDULE-ERR' | translate }}</h3
                  placeholder="{{'PAGES.EDIT-JOB.FIELDS.AT-TIME' | translate}}" type="time" class="example-right-align"
                  [disabled]="jobTypeSelected !== jobType.Recurring">
         </mat-form-field>
+        <mat-form-field>
+          <input [(ngModel)]="jobMaxExecutionTime" [required]="jobTypeSelected !== jobType.Live" name="jobMaxExecutionTime" matInput
+                 placeholder="{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME' | translate}}" type="number" min="0" step="1"
+                 [disabled]="jobTypeSelected === jobType.Live">
+          <span matSuffix>{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME-MINUTES' | translate}}</span>
+          <mat-hint>{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME-HINT' | translate}}</mat-hint>
+        </mat-form-field>
       </mat-card>
     </div>
   </div>

src/app/components/edit-job/edit-job.component.ts

@@ -42,6 +42,7 @@ export class EditJobComponent implements OnInit {
   jobDay = [];
   jobDayOfMonth = [];
   jobTime = '00:00';
+  jobMaxExecutionTime = 0;
 
   months = this.cronService.months;
   days = this.cronService.days;
@@ -89,6 +90,7 @@ export class EditJobComponent implements OnInit {
       this.jobDay = this.job.period.day;
       this.jobDayOfMonth = this.job.period.dayOfMonth;
       this.jobTime = this.job.period.time;
+      this.jobMaxExecutionTime = this.job.getMaxExecutionTimeFormatted();
     });
 
     Promise.resolve().then(() => {
@@ -124,6 +126,7 @@ export class EditJobComponent implements OnInit {
       this.scheduleError = false;
     }
     this.job.setStatus(JobStatus.Active);
+    this.job.setMaxExecutionTime(this.jobMaxExecutionTime);
     this.job.name = this.jobName;
     this.job.description = this.jobDescription;
     this.job.bucket = this.jobBucket;

src/app/components/jobs-list/jobs-list.component.html

@@ -16,9 +16,13 @@ <h2>{{ 'PAGES.JOB-LIST.TITLE' | translate }}</h2>
           <h4 mat-line>{{job.name}}</h4>
           <p mat-line *ngIf="job.type !== jobType.Live">
             <strong>{{ 'PAGES.JOB-LIST.START-DATE' | translate }}:</strong> {{job.getStartDateReadable()}}
-            <span *ngIf="job.type !== jobType.OneTime">
+            <span *ngIf="job.type === jobType.Recurring">
               - <strong>{{ 'PAGES.JOB-LIST.END-DATE' | translate }}:</strong> {{job.getEndDateReadable()}}
             </span>
+            <span *ngIf="job.type !== jobType.Live && job.maxExecutionTime !== 0">
+              - <strong>{{ 'PAGES.JOB-LIST.MAX-EXECUTION-TIME' | translate }}:</strong> {{job.getMaxExecutionTimeFormatted()}}
+              {{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME-MINUTES' | translate}}
+            </span>
             <span *ngIf="job.type !== jobType.OneTime && job.status !== jobStatus.Terminated">
               - <strong>{{ 'PAGES.JOB-LIST.NEXT-RUN' | translate }}: </strong> <span *ngIf="scheduledJobs[job.id]; else no_next_run">{{scheduledJobs[job.id]}}</span>
               <ng-template #no_next_run> {{ 'PAGES.JOB-LIST.NO-NEXT-RUN' | translate }}</ng-template>
@@ -30,6 +34,11 @@ <h4 mat-line>{{job.name}}</h4>
           <div *ngIf="job.isRunning">
             <mat-spinner matTooltip="{{ 'PAGES.JOB-LIST.BACK-RUN' | translate }}" [diameter]="24"></mat-spinner>
           </div>
+          <button *ngIf="job.isRunning && job.type !== jobType.Live" (click)="stopBackupNow(job)"
+                  matTooltip="{{ 'PAGES.JOB-LIST.STOP-BACK-NOW' | translate }}" mat-icon-button class="float-right">
+            <mat-icon color="accent">stop</mat-icon>
+          </button>
+
           <button *ngIf="!job.isRunning && job.type !== jobType.Live" (click)="startBackupManually(job)"
                   matTooltip="{{ 'PAGES.JOB-LIST.BACK-NOW' | translate }}" mat-icon-button class="float-right">
             <mat-icon svgIcon="custom_icon_cloud_up"></mat-icon>

src/app/components/jobs-list/jobs-list.component.ts

@@ -9,6 +9,9 @@ import {MatDialog} from '@angular/material';
 import {JobBackupManuallyComponent} from '../dialogs/job-backup-manually/job-backup-manually.component';
 import {MatIconRegistry} from '@angular/material/icon';
 import {DomSanitizer} from '@angular/platform-browser';
+import {ProcessesHandlerService} from '../../providers/processes-handler.service';
+import {LogService} from '../../providers/log.service';
+import {LogType} from '../../enum/log.type.enum';
 
 @Component({
   selector: 'app-jobs-list',
@@ -28,7 +31,9 @@ export class JobsListComponent implements OnInit {
     public jobScheduler: JobSchedulerService,
     private dialog: MatDialog,
     private matIconRegistry: MatIconRegistry,
-    private domSanitizer: DomSanitizer
+    private domSanitizer: DomSanitizer,
+    private processesHandler: ProcessesHandlerService,
+    private logService: LogService
   ) {
     this.registerIcons();
   }
@@ -59,6 +64,12 @@ export class JobsListComponent implements OnInit {
     event.stopPropagation();
   }
 
+  stopBackupNow(job: Job) {
+    this.logService.printLog(LogType.INFO, 'The job ' +  job.name + ' was stopped manually.');
+    this.processesHandler.killJobProcesses(job.id);
+    event.stopPropagation();
+  }
+
   registerIcons() {
     this.matIconRegistry.addSvgIcon(
       'custom_icon_check',

src/app/components/new-job/new-job.component.html

@@ -87,6 +87,13 @@ <h3 class="mat-error">{{ 'PAGES.EDIT-JOB.FIELDS.SCHEDULE-ERR' | translate }}</h3
                  placeholder="{{'PAGES.EDIT-JOB.FIELDS.AT-TIME' | translate}}" type="time" class="example-right-align"
                  [disabled]="job.type !== jobType.Recurring">
         </mat-form-field>
+        <mat-form-field>
+          <input [(ngModel)]="jobMaxExecutionTimeFormatted" [required]="job.type !== jobType.Live" name="jobMaxExecutionTime" matInput
+                 placeholder="{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME' | translate}}" type="number" min="0" step="1"
+                 [disabled]="job.type === jobType.Live" (ngModelChange)="job.setMaxExecutionTime($event)">
+          <span matSuffix>{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME-MINUTES' | translate}}</span>
+          <mat-hint>{{'PAGES.EDIT-JOB.FIELDS.MAX-EXECUTION-TIME-HINT' | translate}}</mat-hint>
+        </mat-form-field>
       </mat-card>
     </div>
   </div>

src/app/components/new-job/new-job.component.ts

@@ -19,6 +19,7 @@ export class NewJobComponent implements OnInit {
 
   job: Job;
   jobStartDateFormatted: string;
+  jobMaxExecutionTimeFormatted: number;
   jobEndDateFormatted: string;
   filesError: boolean;
   scheduleError: boolean;
@@ -50,6 +51,7 @@ export class NewJobComponent implements OnInit {
     this.appMenuService.changeMenuPage('PAGES.NEW-JOB.MENU');
     this.jobStartDateFormatted = this.job.getStartDateFormatted();
     this.jobEndDateFormatted = this.job.getEndDateFormatted();
+    this.jobMaxExecutionTimeFormatted = this.job.getMaxExecutionTimeFormatted();
   }
 
   saveNewJob() {

src/app/components/s3-explorer/s3-explorer.component.html

@@ -13,6 +13,10 @@ <h2>{{'PAGES.S3-EXPLORER.TITLE' | translate}}</h2>
             <mat-option *ngFor="let bucket of buckets" [value]="bucket.Name">{{bucket.Name}}</mat-option>
           </mat-select>
         </mat-form-field>
+        <div *ngIf="currentBucket !== ''">
+          <span class="dimension-label">{{'PAGES.S3-EXPLORER.BUCKET-SIZE' | translate}}: </span>{{ currentBucketSize }}
+          <span class="dimension-label">{{'PAGES.S3-EXPLORER.BUCKET-OBJECTS' | translate}}: </span>{{ currentBucketNumberOfObjects }}
+        </div>
         <br>
         <p *ngIf="currentPrefix !== ''"><strong>{{'PAGES.S3-EXPLORER.PATH' | translate}}:</strong> {{currentPrefix}}</p>
         <mat-icon class="back-prefix" matTooltip="{{'PAGES.S3-EXPLORER.GO-BACK' | translate}}" (click)="selectKey(backPrefix)" *ngIf="currentPrefix !== ''">

src/app/components/s3-explorer/s3-explorer.component.scss

@@ -0,0 +1,3 @@
+.dimension-label{
+  font-weight: bold;
+}

src/app/components/s3-explorer/s3-explorer.component.ts

@@ -22,6 +22,8 @@ export class S3ExplorerComponent implements OnInit {
   currentFiles = [];
   currentPrefix = '';
   backPrefix = '';
+  currentBucketSize = '--';
+  currentBucketNumberOfObjects = '--';
 
   constructor(
     private appMenuService: AppMenuService,
@@ -52,7 +54,21 @@ export class S3ExplorerComponent implements OnInit {
         this.currentDirs = data.directories;
         this.currentFiles = data.files;
       }
-      this.spinner = false;
+
+      if (prefix === '') {
+        this.aws.getBucketSizeBytes(this.currentBucket, 'StandardStorage').then((size) => {
+          this.currentBucketSize = size;
+
+          this.aws.getBucketNumberOfObjects(this.currentBucket).then((number) => {
+            this.currentBucketNumberOfObjects = number;
+            this.spinner = false;
+          });
+
+        });
+      } else {
+        this.spinner = false;
+      }
+
     });
   }