Merge pull request #366 from jacobwilliams/speed

Speed

Author: jacobwilliams

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-  - 3.5
+  - 3.6
 
 sudo: false
 

src/json_parameters.F90

@@ -113,17 +113,20 @@ module json_parameters
         !! 6 = sign + leading 0 + decimal + 'E' + exponent sign + 1 extra
     character(kind=CDK,len=*),parameter :: int_fmt  = '(ss,I0)' !! minimum width format for integers
 
-    integer(IK),parameter :: max_integer_str_len = 256 !! maximum string length of an integer.
-                                                       !! This is totally arbitrary (any way
-                                                       !! to get the compiler to tell us this?)
+    integer(IK),parameter :: max_integer_str_len = 256_IK !! maximum string length of an integer.
+                                                          !! This is totally arbitrary (any way
+                                                          !! to get the compiler to tell us this?)
 
-    integer(IK),parameter :: chunk_size = 100_IK  !! for allocatable strings: allocate chunks of this size
+    integer(IK),parameter :: chunk_size = 256_IK  !! for allocatable strings: allocate chunks of this size
     integer(IK),parameter :: unit2str = -1_IK  !! unit number to cause stuff to be
                                                !! output to strings rather than files.
                                                !! See 9.5.6.12 in the F2003/08 standard
+    character(kind=CK,len=*),parameter :: blank_chunk = repeat(space, chunk_size) !! a blank string
 
     integer(IK),parameter :: seq_chunk_size = 256_IK !! chunk size for reading sequential files
 
+    integer(IK),parameter :: stream_chunk_size = 256_IK !! chunk size for reading stream files
+
     integer(IK),parameter :: pushed_char_size = 10_IK !! size for `pushed_char`
                                                       !! array in [[json_core(type)]]
 

src/json_string_utilities.F90

@@ -125,14 +125,17 @@ subroutine string_to_integer(str,ival,status_ok)
 
     ! Compute how many digits we need to read
     ndigits = 2*len_trim(str)
-    ndigits_digits = floor(log10(real(ndigits)))+1
-    allocate(character(kind=CDK,len=ndigits_digits) :: digits)
-    write(digits,'(I0)') ndigits !gfortran will have a runtime error with * edit descriptor here
-    ! gfortran bug: '*' edit descriptor for ISO_10646 strings does bad stuff.
-    read(str,'(I'//trim(digits)//')',iostat=ierr) ival   !string to integer
-
-    ! error check:
-    status_ok = (ierr==0)
+    if (ndigits/=0) then
+        ndigits_digits = floor(log10(real(ndigits)))+1
+        allocate(character(kind=CDK,len=ndigits_digits) :: digits)
+        write(digits,'(I0)') ndigits !gfortran will have a runtime error with * edit descriptor here
+        ! gfortran bug: '*' edit descriptor for ISO_10646 strings does bad stuff.
+        read(str,'(I'//trim(digits)//')',iostat=ierr) ival   !string to integer
+        ! error check:
+        status_ok = (ierr==0)
+    else
+        status_ok = .false.
+    end if
     if (.not. status_ok) ival = 0_IK
 
     end subroutine string_to_integer
@@ -342,7 +345,7 @@ subroutine escape_string(str_in, str_out, escape_solidus)
             c = str_in(i:i)    !get next character in the input string
 
             !if the string is not big enough, then add another chunk:
-            if (ipos+3>len(str_out)) str_out = str_out // repeat(space, chunk_size)
+            if (ipos+3>len(str_out)) str_out = str_out // blank_chunk
 
             select case(c)
             case(backslash)

src/json_value_module.F90

@@ -256,6 +256,12 @@ module json_value_module
                                                   !! (both escaped and unescaped versions are still
                                                   !! valid in all cases).
 
+        integer :: ichunk = 0 !! index in `chunk` for [[pop_char]]
+                              !! when `use_unformatted_stream=True`
+        integer :: filesize = 0 !! the file size when when `use_unformatted_stream=True`
+        character(kind=CK,len=:),allocatable :: chunk   !! a chunk read from a stream file
+                                                        !! when `use_unformatted_stream=True`
+
         contains
 
         private
@@ -918,6 +924,11 @@ subroutine json_initialize(me,verbose,compact_reals,&
     me%char_count   = 0
     me%line_count   = 1
     me%ipos         = 1
+    if (use_unformatted_stream) then
+        me%filesize = 0
+        me%ichunk   = 0
+        me%chunk    = repeat(' ', stream_chunk_size) ! default chunk size
+    end if
 
 #ifdef USE_UCS4
     ! reopen stdout and stderr with utf-8 encoding
@@ -8755,6 +8766,11 @@ subroutine json_parse_file(json, file, p, unit)
 
     if (istat==0) then
 
+        if (use_unformatted_stream) then
+            ! save the file save to be read:
+            inquire(unit=iunit, size=json%filesize, iostat=istat)
+        end if
+
         ! create the value and associate the pointer
         call json_value_create(p)
 
@@ -9110,6 +9126,7 @@ recursive subroutine parse_value(json, unit, str, value)
         !the routine is being called incorrectly.
         if (.not. associated(value)) then
             call json%throw_exception('Error in parse_value: value pointer not associated.')
+            return
         end if
 
         ! pop the next non whitespace character off the file
@@ -9957,7 +9974,7 @@ subroutine parse_string(json, unit, str, string)
     character(kind=CK,len=:),allocatable :: error_message !! for string unescaping
 
     !at least return a blank string if there is a problem:
-    string = repeat(space, chunk_size)
+    string = blank_chunk
 
     if (.not. json%exception_thrown) then
 
@@ -9982,7 +9999,7 @@ subroutine parse_string(json, unit, str, string)
             else
 
                 !if the string is not big enough, then add another chunk:
-                if (ip>len(string)) string = string // repeat(space, chunk_size)
+                if (ip>len(string)) string = string // blank_chunk
 
                 !append to string:
                 string(ip:ip) = c
@@ -10098,7 +10115,7 @@ subroutine parse_number(json, unit, str, value)
 
     if (.not. json%exception_thrown) then
 
-        tmp = repeat(space, chunk_size)
+        tmp = blank_chunk
         ip = 1
         first = .true.
         is_integer = .true.  !assume it may be an integer, unless otherwise determined
@@ -10122,7 +10139,7 @@ subroutine parse_number(json, unit, str, value)
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
@@ -10132,15 +10149,15 @@ subroutine parse_number(json, unit, str, value)
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
                 case(CK_'0':CK_'9')    !valid characters for numbers
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
@@ -10185,7 +10202,7 @@ end subroutine parse_number
 !@note This routine ignores non-printing ASCII characters
 !      (`iachar<=31`) that are in strings.
 
-    recursive subroutine pop_char(json,unit,str,skip_ws,skip_comments,eof,popped)
+    subroutine pop_char(json,unit,str,skip_ws,skip_comments,eof,popped)
 
     implicit none
 
@@ -10239,16 +10256,41 @@ recursive subroutine pop_char(json,unit,str,skip_ws,skip_comments,eof,popped)
 
                     !read the next character:
                     if (use_unformatted_stream) then
-                        read(unit=unit,pos=json%ipos,iostat=ios) c
+
+                        ! in this case, we read the file in chunks.
+                        ! if we already have the character we need,
+                        ! then get it from the chunk. Otherwise,
+                        ! read in another chunk.
+                        if (json%ichunk<1) then
+                            ! read in a chunk:
+                            json%ichunk = 0
+                            if (json%filesize<json%ipos+len(json%chunk)-1) then
+                                ! for the last chunk, we resize
+                                ! it to the correct size:
+                                json%chunk = repeat(' ', json%filesize-json%ipos+1)
+                            end if
+                            read(unit=unit,pos=json%ipos,iostat=ios) json%chunk
+                        else
+                            ios = 0
+                        end if
+                        json%ichunk = json%ichunk + 1
+                        if (json%ichunk>len(json%chunk)) then
+                            ! check this just in case
+                            ios = IOSTAT_END
+                        else
+                            ! get the next character from the chunk:
+                            c = json%chunk(json%ichunk:json%ichunk)
+                            if (json%ichunk==len(json%chunk)) then
+                                json%ichunk = 0 ! reset for next chunk
+                            end if
+                        end if
+
                     else
+                        ! a formatted read:
                         read(unit=unit,fmt='(A1)',advance='NO',iostat=ios) c
                     end if
                     json%ipos = json%ipos + 1
 
-                    !....note: maybe try read the file in chunks...
-                    !.... or use asynchronous read with double buffering
-                    !     (see Modern Fortran: Style and Usage)
-
                 else    !read from the string
 
                     str_len = len(str)   !length of the string
@@ -10339,7 +10381,8 @@ subroutine push_char(json,c)
 
             !in this case, c is ignored, and we just
             !decrement the stream position counter:
-            json%ipos = json%ipos - 1
+            json%ipos   = json%ipos - 1
+            json%ichunk = json%ichunk - 1
 
         else