version 2.1.3 - process improvements

see changelog

Author: WillTheFarmer

.github/CHANGELOG.md

@@ -5,6 +5,7 @@
 - version 2.1.0 - 12/09/2024 - request_log_id functionality
 - version 2.1.1 - 12/11/2024 - rename column timeStamp to logged - add 4 indexes and 10 views
 - version 2.1.2 - 12/20/2024 - several improvements
+- version 2.1.3 - 12/27/2024 - process improvements
 - [1.0.1] apache_logs.error_systemCodeID corrected line - INTO logsystemCode to INTO logsystemCodeID
 - [1.0.1] remove debugging - SELECT statement from apache_logs.process_access_import, process_error_import & normalize_useragent.
 - [1.0.1] remove whitespace and commented out old code on all stored programs
@@ -69,4 +70,8 @@
 - [2.1.2] modify `process_access_parse` and `process_error_parse` - WHERE CLAUSE for parameter `ALL` to select ONLY completed LOAD processes.  
 - [2.1.2] reformatted SQL statements in all 66 schema views for code standardization in SQL files used to create `apacheLogs2MySQL.sql`
 - [2.1.2] modify all 11 `access_ua_` views SQL statements `FROM apache_logs.access_log_ua ln INNER JOIN apache_logs.access_log_useragent lua INNER JOIN apache_logs.access_log`
-- [2.1.2] created new `entity_relationship_diagram.png` to reflect database changes.
+- [2.1.2] created new `entity_relationship_diagram.png` to reflect database changes.
+- [2.1.3] modify `access_log` and `error_log` TABLES reordering COLUMNS to improve database design readability.
+- [2.1.3] modify `normalize_useragent` to removed first parameter restriction. Any string 8 characters are more can be passed.
+- [2.1.3] modify `apacheLogs2MySQL.py` add `completed` COLUMN to UPDATE statement to fix processing `process_access_import` and `process_error_import` with `ALL` parameter.
+- [2.1.3] modify `call_processes.sql` adding more comments to better describe options and parameters and overall processing.

.github/README.md

@@ -225,7 +225,7 @@ Run polling module from PM2:
 ```
 pm2 start watch4logs.py
 ```
-Run MySQL Stored Procedures from Command Line Client or Workbench:
+Run MySQL Stored Procedures from Command Line Client or GUI Database Tool:
 
 Passing 'ALL' as second parameter processes ALL files & records based process_status. 
 This can be multiple importloadid values. Passing an importloadid value as a STRING processes ONLY files & records related to that importloadid.

.github/call_processes.sql

@@ -5,8 +5,11 @@ process_access_parse - Parsing into proper columns. Apache Access formats use %r
 process_access_import - normalization of parsed LOAD DATA table into 9 error tables & 3 comon log tables shared with error logs.
 normalize_useragent - normalized Python parsed userAgent TABLE into 11 tables
 Each Stored Procudure has 2 parameters. 
-- IN in_processName VARCHAR(100) - This indicates the LogFormat to process.
+- IN in_processName VARCHAR(100) - This indicates the LogFormat to process. 
+NOTE: For normalize_useragent parameter can be any string >= 8 characters
 - IN in_importLoadID VARCHAR(20) - This indicates to importloadid to process. Valid values 'ALL' or a value converted to INTEGER=importloadid   
+NOTE: if in_importLoadID='ALL' ONLY importloadID records with import_load TABLE "completed" COLUMN NOT NULL will be processed. This is to avoid
+interfering with Python client modules uploading files at same time as server STORED PROCEDURES executing.
 
 import_file TABLE - record of every access & error log file processed by Python processFiles function. Each LOAD DATA table has importfileid COLUMN.
 import_load TABLE  - record for every execution of Python process. Each record contains information on LogFormat log files processed. 
@@ -19,8 +22,19 @@ by Python apacheLogs2MySQL.py processLogs function Or only LOAD DATA can be exec
 
 LOAD DATA stage tables - load_access_combined, load_access_csv2mysql, load_access_vhost, load_error_default have a process_status COLUMN.
 process_status=0 - LOAD DATA tables loaded with raw log data
-process_status=1 - process_error_parse and process_access_parse executed on record
-process_status=2 - process_error_import and process_access_import executed on record
+process_status=1 - process_error_parse or process_access_parse executed on record
+process_status=2 - process_error_import or process_access_import executed on record
+
+If importing multiple domains check before executing process_error_import or process_access_import STORED PROCEDURES. 
+In order to filter and report data properly this SELECT statement should return NO RECORDS. 
+ - SELECT * FROM apache_logs.import_file WHERE server_name IS NULL;
+
+import_file TABLE UPDATES should execute after _parse STORED PROCEDURES if importing multiple formats with servername (%v). Parsing will fill-in columns.
+if not using environment variables to SET ServerName and ServerPort in Python and no formats contain %v Format String UPDATES can be executed before Parsing.
+import_file TABLE UPDATES must be executed before _import STORED PROCEDURES. Below are examples of possible UPDATE STATEMENTS based on imported log file name.
+ - UPDATE apache_logs.import_file SET server_name='farmfreshsoftware.com', server_port=443 WHERE server_name IS NULL AND name LIKE '%farmfreshsoftware%';
+ - UPDATE apache_logs.import_file SET server_name='farmwork.app', server_port=443 WHERE server_name IS NULL AND name LIKE '%farmwork%';
+ - UPDATE apache_logs.import_file SET server_name='ip255-255-255-255.us-east.com', server_port=443 WHERE server_name IS NULL AND name LIKE '%error%';
 
 Commands below execute each Stored Procudure and process ALL importloadid based on process_status. Each VPS creates importloadid each time executed.
 */
@@ -38,4 +52,4 @@ CALL process_access_import('vhost','ALL');
 CALL process_access_parse('csv2mysql','ALL'); 
 CALL process_access_import('csv2mysql','ALL'); 
 
-CALL normalize_useragent('MySQL Workbench','ALL');
+CALL normalize_useragent('Any Process Name','ALL');

apacheLogs2MySQL.py

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# version 2.1.2 - 12/20/2024 - several improvements - see changelog
+# version 2.1.3 - 12/27/2024 - process improvements - see changelog
 #
 # Copyright 2024 Will Raymond <[email protected]>
 #
@@ -651,6 +651,7 @@ def processLogs():
                   ', csv2mysqlLogProcessed=' + str(csv2mysqlFileProcessed) + 
                   ', userAgentProcessed=' + str(useragentFileProcessed) + 
                   ', errorOccurred=' + str(processError) + 
+                  ', completed=now()' + 
                   ', processSeconds=' + str(processSeconds) + ' WHERE id=' + str(importLoadID) +';')
     importLoadCursor = conn.cursor()
     try:

apacheLogs2MySQL.sql

@@ -10,7 +10,7 @@
 -- # See the License for the specific language governing permissions and
 -- # limitations under the License.
 -- #
--- # version 2.1.2 - 12/20/2024 - several improvements - see changelog
+-- # version 2.1.3 - 12/27/2024 - process improvements - see changelog
 -- #
 -- # Copyright 2024 Will Raymond <[email protected]>
 -- #
@@ -111,8 +111,13 @@ DROP TABLE IF EXISTS `access_log`;
 /*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `access_log` (
   `id` int NOT NULL AUTO_INCREMENT,
-  `importfileid` int NOT NULL,
-  `logged` datetime DEFAULT NULL,
+  `logged` datetime NOT NULL,
+  `servernameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_servername',
+  `serverportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_serverport',
+  `clientnameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientname',
+  `clientportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientport',
+  `refererid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_referer',
+  `requestlogid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_requestlogid',
   `bytes_received` int NOT NULL,
   `bytes_sent` int NOT NULL,
   `bytes_transferred` int NOT NULL,
@@ -127,14 +132,9 @@ CREATE TABLE `access_log` (
   `reqqueryid` int DEFAULT NULL,
   `remoteuserid` int DEFAULT NULL,
   `remotelognameid` int DEFAULT NULL,
-  `refererid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_referer',
-  `clientnameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientname',
-  `clientportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientport',
-  `servernameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_servername',
-  `serverportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_serverport',
-  `requestlogid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_requestlogid',
   `cookieid` int DEFAULT NULL,
   `useragentid` int DEFAULT NULL,
+  `importfileid` int NOT NULL,
   `added` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
   KEY `F_access_reqstatus` (`reqstatusid`),
@@ -1676,8 +1676,13 @@ DROP TABLE IF EXISTS `error_log`;
 /*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `error_log` (
   `id` int NOT NULL AUTO_INCREMENT,
-  `importfileid` int NOT NULL,
   `logged` datetime NOT NULL,
+  `servernameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_servername',
+  `serverportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_serverport',
+  `clientnameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientname',
+  `clientportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientport',
+  `refererid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_referer',
+  `requestlogid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_requestlogid',
   `loglevelid` int DEFAULT NULL,
   `moduleid` int DEFAULT NULL,
   `processid` int DEFAULT NULL,
@@ -1687,12 +1692,7 @@ CREATE TABLE `error_log` (
   `systemcodeid` int DEFAULT NULL,
   `systemmessageid` int DEFAULT NULL,
   `logmessageid` int DEFAULT NULL,
-  `refererid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_referer',
-  `clientnameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientname',
-  `clientportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_clientport',
-  `servernameid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_servername',
-  `serverportid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_serverport',
-  `requestlogid` int DEFAULT NULL COMMENT 'Access & Error shared normalization table - log_requestlogid',
+  `importfileid` int NOT NULL,
   `added` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
   KEY `F_error_level` (`loglevelid`),
@@ -2197,12 +2197,12 @@ CREATE TABLE `import_client` (
   `deviceid` varchar(200) NOT NULL,
   `login` varchar(200) NOT NULL,
   `expandUser` varchar(200) NOT NULL,
-  `platformSystem` varchar(100) DEFAULT NULL,
-  `platformNode` varchar(100) DEFAULT NULL,
-  `platformRelease` varchar(100) DEFAULT NULL,
-  `platformVersion` varchar(150) DEFAULT NULL,
-  `platformMachine` varchar(100) DEFAULT NULL,
-  `platformProcessor` varchar(150) DEFAULT NULL,
+  `platformSystem` varchar(100) NOT NULL,
+  `platformNode` varchar(100) NOT NULL,
+  `platformRelease` varchar(100) NOT NULL,
+  `platformVersion` varchar(150) NOT NULL,
+  `platformMachine` varchar(100) NOT NULL,
+  `platformProcessor` varchar(150) NOT NULL,
   `added` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Table keeps track of all application Windows, Linux and Mac clients loading logs to server application and long with logon and IP address information. It is important to know who is loading logs.';
@@ -2314,7 +2314,7 @@ CREATE TABLE `import_format` (
 
 LOCK TABLES `import_format` WRITE;
 /*!40000 ALTER TABLE `import_format` DISABLE KEYS */;
-INSERT INTO `import_format` VALUES (1,'common',NULL,'2024-12-20 16:31:09'),(2,'combined',NULL,'2024-12-20 16:31:09'),(3,'vhost',NULL,'2024-12-20 16:31:09'),(4,'csc2mysql',NULL,'2024-12-20 16:31:09'),(5,'error_default',NULL,'2024-12-20 16:31:09'),(6,'error_vhost',NULL,'2024-12-20 16:31:09');
+INSERT INTO `import_format` VALUES (1,'common',NULL,'2024-12-27 12:55:32'),(2,'combined',NULL,'2024-12-27 12:55:32'),(3,'vhost',NULL,'2024-12-27 12:55:32'),(4,'csc2mysql',NULL,'2024-12-27 12:55:32'),(5,'error_default',NULL,'2024-12-27 12:55:32'),(6,'error_vhost',NULL,'2024-12-27 12:55:32');
 /*!40000 ALTER TABLE `import_format` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -2410,12 +2410,12 @@ DROP TABLE IF EXISTS `import_server`;
 /*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `import_server` (
   `id` int NOT NULL AUTO_INCREMENT,
-  `dbuser` varchar(255) DEFAULT NULL,
-  `dbhost` varchar(255) DEFAULT NULL,
-  `dbversion` varchar(255) DEFAULT NULL,
-  `dbsystem` varchar(255) DEFAULT NULL,
-  `dbmachine` varchar(255) DEFAULT NULL,
-  `serveruuid` varchar(255) DEFAULT NULL,
+  `dbuser` varchar(255) NOT NULL,
+  `dbhost` varchar(255) NOT NULL,
+  `dbversion` varchar(255) NOT NULL,
+  `dbsystem` varchar(255) NOT NULL,
+  `dbmachine` varchar(255) NOT NULL,
+  `serveruuid` varchar(255) NOT NULL,
   `added` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Table for keeping track of log processing servers and login information.';
@@ -4381,8 +4381,8 @@ BEGIN
   IF CONVERT(in_importLoadID, UNSIGNED) = 0 AND in_importLoadID != 'ALL' THEN
     SIGNAL SQLSTATE '22003' SET MESSAGE_TEXT = 'Invalid parameter value for in_importLoadID. Must be convert to number or be ALL';
   END IF;
-  IF FIND_IN_SET(in_processName, "default,MySQL WorkBench,Python Processed") = 0 THEN
-    SIGNAL SQLSTATE '22003' SET MESSAGE_TEXT = 'Invalid parameter value for in_processName. Must be default,MySQL WorkBench OR Python Processed';
+  IF LENGTH(in_processName) < 8 THEN
+    SIGNAL SQLSTATE '22003' SET MESSAGE_TEXT = 'Invalid parameter value for in_processName. Must be minimum of 8 characters';
   END IF;
 	IF NOT CONVERT(in_importLoadID, UNSIGNED) = 0 THEN
 		SET importLoad_ID = CONVERT(in_importLoadID, UNSIGNED);

watch4logs.py

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# version 2.1.2 - 12/20/2024 - several improvements - see changelog
+# version 2.1.3 - 12/27/2024 - process improvements - see changelog
 #
 # Copyright 2024 Will Raymond <[email protected]>
 #