harden paramdict parsing and typed access (#6983)

Co-authored-by: Nathan Keys <nathan.keys@pm.me>
Co-authored-by: beilzx <2069360659@qq.com>

Author: 3 people

docs/developer-guide/param-and-model-file-structure.md

@@ -18,6 +18,11 @@ Softmax       softmax  1 1 fc prob 0=0
 * layer count : count of the layer line follows, should be exactly the count of all layer names
 * blob count : count of all blobs, usually greater than or equals to the layer count
 ### layer line
+
+each layer must occupy exactly one physical line, including all blob names and parameters; do not split a layer across lines or put multiple layers on the same line
+
+Custom `DataReader::scan()` implementations and C API `ncnn_datareader_t::scan` callbacks must follow scanf conversion and input consumption rules, including field widths and scansets. Parameter parsing uses `%1023[^\r\n]` to read up to 1023 characters without skipping leading whitespace or consuming CR/LF, repeating the scan for longer lines. A successful scanset conversion must append a null terminator and return 1. Returning 0 for an unsupported format can be interpreted as an empty parameter list and silently select default parameter values.
+
 ```
 [layer type] [layer name] [input count] [output count] [input blobs] [output blobs] [layer specific params]
 ```
@@ -36,13 +41,21 @@ key index should be unique in each layer line, pair can be omitted if the defaul
 
 the meaning of existing param key index can be looked up at [operation-param-weight-table](operation-param-weight-table)
 
-* integer or float key : index 0 ~ 19
+* integer or float key : index 0 ~ 31
 * integer value : int
 * float value : float
-* integer array or float array key : -23300 minus index 0 ~ 19
+* integer array or float array key : -23300 minus index 0 ~ 31
 * integer array value : [array size],int,int,...,int
 * float array value : [array size],float,float,...,float
 
+Use a decimal point or exponent when generating floating-point scalar values, including integral values, for example `1=6.0`, `1=6e0`, or `1=0.0`. When loading text parameters, the float getter also converts integer spellings such as `1=6` and `1=0` to `6.0f` and `0.0f`. The int getter does not convert floating-point parameters to integers.
+
+Keep floating-point spellings when converting models with `ncnn2mem`: binary scalar parameters do not retain integer/float type tags, and the converter writes integer spellings as integer bit patterns without this numeric conversion.
+
+Use a decimal point or exponent for every element of a floating-point array, including integral values, for example `-23303=2,1.0,2.0`. Mixed integer and float element spellings within an array are not defined by the format.
+
+A zero-length array such as `-23300=0` explicitly supplies an empty array. Array getters return that empty array even when a nonempty default is supplied; omitting the parameter returns the default.
+
 In modern ncnn param file
 
 * array could be represented as `3=2.0,3.0` that is much more human friendly

src/c_api.h

@@ -224,6 +224,9 @@ struct NCNN_EXPORT __ncnn_datareader_t
     void* pthis;
 
 #if NCNN_STRING
+    /* follow scanf conversion and input consumption rules, including field widths and scansets */
+    /* %1023[^\r\n] must preserve leading whitespace and leave CR/LF unread */
+    /* append a null terminator on a successful scanset conversion and return 1 */
     int (*scan)(ncnn_datareader_t dr, const char* format, void* p);
 #endif /* NCNN_STRING */
     size_t (*read)(ncnn_datareader_t dr, void* buf, size_t size);

src/datareader.cpp

@@ -3,6 +3,7 @@
 
 #include "datareader.h"
 
+#include <ctype.h>
 #include <string.h>
 
 namespace ncnn {
@@ -187,23 +188,12 @@ int DataReaderFromAndroidAsset::scan(const char* format, void* p) const
     std::string line;
     {
         off64_t remain_length = AAsset_getRemainingLength64(d->asset);
-        const char* newline_pos;
-        if (remain_length > 1 && ((const char*)d->mem)[0] == '\n')
-        {
-            // skip the leading newline
-            // however, it is fine to create "\nXYZ 123 abc" as sscanf will skip the leading newline silently
-            newline_pos = (const char*)memchr((const char*)d->mem + 1, '\n', remain_length - 1);
-        }
-        else if (remain_length > 2 && ((const char*)d->mem)[0] == '\r' && ((const char*)d->mem)[1] == '\n')
-        {
-            // skip the leading newline
-            // however, it is fine to create "\r\nXYZ 123 abc" as sscanf will skip the leading newline silently
-            newline_pos = (const char*)memchr((const char*)d->mem + 2, '\n', remain_length - 2);
-        }
-        else
-        {
-            newline_pos = (const char*)memchr((const char*)d->mem, '\n', remain_length);
-        }
+        // include all leading whitespace and the following line in the buffer
+        // sscanf decides whether the conversion consumes that whitespace
+        size_t offset = 0;
+        while (offset < (size_t)remain_length && isspace(d->mem[offset]))
+            offset++;
+        const char* newline_pos = (const char*)memchr(d->mem + offset, '\n', (size_t)remain_length - offset);
 
         size_t line_length = newline_pos ? newline_pos - (const char*)d->mem : (size_t)remain_length;
         line = std::string((const char*)d->mem, line_length);

src/datareader.h

@@ -26,6 +26,9 @@ class NCNN_EXPORT DataReader
 
 #if NCNN_STRING
     // parse plain param text
+    // follow scanf conversion and input consumption rules, including field widths and scansets
+    // %1023[^\r\n] reads at most 1023 characters without skipping whitespace or consuming CR/LF
+    // append a null terminator on a successful scanset conversion
     // return 1 if scan success
     virtual int scan(const char* format, void* p) const;
 #endif // NCNN_STRING