sim

Author: nihui

docs/developer-guide/param-and-model-file-structure.md

@@ -18,6 +18,9 @@ Softmax       softmax  1 1 fc prob 0=0
 * layer count : count of the layer line follows, should be exactly the count of all layer names
 * blob count : count of all blobs, usually greater than or equals to the layer count
 ### layer line
+
+each layer must occupy exactly one physical line, including all blob names and parameters; do not split a layer across lines or put multiple layers on the same line
+
 ```
 [layer type] [layer name] [input count] [output count] [input blobs] [output blobs] [layer specific params]
 ```

src/paramdict.cpp

@@ -359,34 +359,71 @@ static bool vstr_to_float(const char* p, float& value)
     return true;
 }
 
-static int scan_numeric_value(const DataReader& dr, char vstr[128], bool comma = false)
+static bool param_space(char c)
 {
-    if (dr.scan(comma ? ",%127[^, \t\r\n\v\f]" : "%127[^, \t\r\n\v\f]", vstr) != 1)
-        return 0;
+    return c == ' ' || c == '\t' || c == '\v' || c == '\f';
+}
 
-    // a field-width limit must not silently split a numeric token
-    char extra[2];
-    if (strlen(vstr) == 127 && dr.scan("%1[^, \t\r\n\v\f]", extra) == 1)
-        return -1;
+static int scan_numeric_value(const char*& p, char vstr[128], bool comma = false)
+{
+    if (comma)
+    {
+        if (*p != ',')
+            return 0;
+        p++;
+    }
 
-    return 1;
+    int len = 0;
+    while (*p && *p != ',' && !param_space(*p))
+    {
+        if (len == 127)
+            return -1;
+        vstr[len++] = *p++;
+    }
+    vstr[len] = '\0';
+    return len > 0 ? 1 : 0;
 }
 
 int ParamDict::load_param(const DataReader& dr)
 {
     clear();
 
-    // require '=' separately: a successful numeric conversion alone does not validate "id="
-    char idstr[16];
-    while (dr.scan(" %15[-+0123456789]", idstr) == 1)
+    // each layer occupies one line
+    // leave the newline for the next layer header scan
+    char line[1024] = {0};
+    std::vector<char> long_line;
+    while (dr.scan("%1023[^\r\n]", line) == 1)
     {
+        const size_t len = strlen(line);
+        if (long_line.empty() && len < sizeof(line) - 1)
+            break;
+        long_line.insert(long_line.end(), line, line + len);
+        if (len < sizeof(line) - 1)
+            break;
+    }
+    if (!long_line.empty())
+        long_line.push_back('\0');
+    const char* p = long_line.empty() ? line : long_line.data();
+
+    while (1)
+    {
+        while (param_space(*p))
+            p++;
+        if (!*p)
+            break;
+
+        char idstr[16];
+        int idlen = 0;
+        while ((*p == '-' || *p == '+' || (*p >= '0' && *p <= '9')) && idlen < 15)
+            idstr[idlen++] = *p++;
+        idstr[idlen] = '\0';
         int id;
-        char delimiter[2];
-        if (!vstr_to_int(idstr, id) || dr.scan("%1[=]", delimiter) != 1)
+        if (!vstr_to_int(idstr, id) || *p != '=')
         {
             NCNN_LOGE("ParamDict invalid parameter id or missing equals sign");
             return -1;
         }
+        p++;
 
         const bool old_array = id <= -23300;
         if (old_array)
@@ -402,7 +439,7 @@ int ParamDict::load_param(const DataReader& dr)
         {
             char vstr[128];
             int len;
-            if (scan_numeric_value(dr, vstr) != 1 || !vstr_to_int(vstr, len) || !valid_array_length((size_t)len))
+            if (scan_numeric_value(p, vstr) != 1 || !vstr_to_int(vstr, len) || !valid_array_length((size_t)len))
             {
                 NCNN_LOGE("ParamDict invalid array length (id=%d)", id);
                 return -1;
@@ -418,7 +455,7 @@ int ParamDict::load_param(const DataReader& dr)
             bool is_float = false;
             for (int j = 0; j < len; j++)
             {
-                if (scan_numeric_value(dr, vstr, true) != 1)
+                if (scan_numeric_value(p, vstr, true) != 1)
                 {
                     NCNN_LOGE("ParamDict read array element failed");
                     return -1;
@@ -433,7 +470,7 @@ int ParamDict::load_param(const DataReader& dr)
                 }
             }
             // extra elements are not a new parameter or the next layer
-            if (dr.scan("%1[,]", delimiter) == 1)
+            if (*p == ',')
             {
                 NCNN_LOGE("ParamDict array length mismatch (id=%d)", id);
                 return -1;
@@ -444,26 +481,25 @@ int ParamDict::load_param(const DataReader& dr)
             continue;
         }
 
-        char first[2];
-        if (dr.scan("%1[\"a-zA-Z]", first) == 1)
+        if (*p == '\"' || (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z'))
         {
             char text[256] = {0};
-            const bool quoted = first[0] == '\"';
+            const bool quoted = *p == '\"';
+            if (quoted)
+                p++;
+            int len = 0;
+            while (*p && (quoted ? *p != '\"' : !param_space(*p)) && len < 255)
+                text[len++] = *p++;
             if (quoted)
             {
-                dr.scan("%255[^\"\r\n]", text);
-                if (dr.scan("%1[\"]", delimiter) != 1)
+                if (*p != '\"')
                 {
                     NCNN_LOGE("ParamDict unterminated or too long string (id=%d)", id);
                     return -1;
                 }
+                p++;
             }
-            else
-            {
-                text[0] = first[0];
-                dr.scan("%254[^ \t\r\n\v\f]", text + 1);
-            }
-            if (dr.scan("%1[^ \t\r\n\v\f]", delimiter) == 1)
+            if (*p && !param_space(*p))
             {
                 NCNN_LOGE("ParamDict invalid string suffix or string too long (id=%d)", id);
                 return -1;
@@ -475,7 +511,7 @@ int ParamDict::load_param(const DataReader& dr)
         }
 
         char vstr[128];
-        if (scan_numeric_value(dr, vstr) != 1)
+        if (scan_numeric_value(p, vstr) != 1)
         {
             NCNN_LOGE("ParamDict read value failed");
             return -1;
@@ -490,27 +526,27 @@ int ParamDict::load_param(const DataReader& dr)
             return -1;
         }
 
-        if (dr.scan("%1[,]", delimiter) == 1)
+        if (*p == ',')
         {
-            Mat values(16);
-            if (values.empty())
-            {
-                NCNN_LOGE("ParamDict array allocation failed (id=%d)", id);
-                return -1;
-            }
+            p++;
+            // keep short arrays on the stack until their final Mat allocation
+            unsigned char local_values[16 * sizeof(float)];
+            Mat values;
+            unsigned char* data = local_values;
+            int capacity = 16;
             int len = 1;
             if (is_float)
-                ((float*)values)[0] = f;
+                memcpy(data, &f, sizeof(float));
             else
-                ((int*)values)[0] = i;
+                memcpy(data, &i, sizeof(int));
 
             while (1)
             {
-                const int nscan = scan_numeric_value(dr, vstr);
+                const int nscan = scan_numeric_value(p, vstr);
                 // a trailing comma is the established syntax for a one-element array
                 if (nscan == 0)
                 {
-                    if (dr.scan("%1[,]", delimiter) == 1)
+                    if (*p == ',')
                     {
                         NCNN_LOGE("ParamDict missing array element (id=%d, index=%d)", id, len);
                         return -1;
@@ -522,26 +558,29 @@ int ParamDict::load_param(const DataReader& dr)
                     NCNN_LOGE("ParamDict invalid array element (id=%d, index=%d)", id, len);
                     return -1;
                 }
-                if (len == values.w)
+                if (len == capacity)
                 {
-                    const size_t capacity = std::min((size_t)values.w * 2, max_array_length());
-                    Mat grown((int)capacity);
+                    const size_t next_capacity = std::min((size_t)capacity * 2, max_array_length());
+                    Mat grown((int)next_capacity);
                     if (grown.empty())
                     {
                         NCNN_LOGE("ParamDict array allocation failed (id=%d)", id);
                         return -1;
                     }
-                    memcpy(grown.data, values.data, (size_t)len * sizeof(float));
+                    memcpy(grown.data, data, (size_t)len * sizeof(float));
                     values = grown;
+                    data = (unsigned char*)values.data;
+                    capacity = (int)next_capacity;
                 }
                 if (is_float)
-                    ((float*)values)[len] = f;
+                    memcpy(data + (size_t)len * sizeof(float), &f, sizeof(float));
                 else
-                    ((int*)values)[len] = i;
+                    memcpy(data + (size_t)len * sizeof(int), &i, sizeof(int));
                 len++;
 
-                if (dr.scan("%1[,]", delimiter) != 1)
+                if (*p != ',')
                     break;
+                p++;
             }
 
             if (len == values.w)
@@ -556,7 +595,7 @@ int ParamDict::load_param(const DataReader& dr)
                     NCNN_LOGE("ParamDict array allocation failed (id=%d)", id);
                     return -1;
                 }
-                memcpy(v.data, values.data, (size_t)len * sizeof(float));
+                memcpy(v.data, data, (size_t)len * sizeof(float));
                 d->params[id].v = v;
             }
             d->params[id].type = is_float ? 6 : 5;

tests/test_paramdict.cpp

@@ -839,7 +839,7 @@ static int test_paramdict_invalid_text()
 static int test_paramdict_text_boundaries()
 {
     ParamDictTest pd;
-    const char* text = "0=-2147483648\t1=2147483647\r\n2=\"\" 3=\" \" 4=1.0,2.0,-3.0 5=7, -23306=2,1.0,2.0 -23307=0 8=0e9999999999 9=1e-9999999999 10=.5 11=4294967296.0";
+    const char* text = "0=-2147483648\t1=2147483647 2=\"\" 3=\" \" 4=1.0,2.0,-3.0 5=7, -23306=2,1.0,2.0 -23307=0 8=0e9999999999 9=1e-9999999999 10=.5 11=4294967296.0";
     if (check_text_result(text, true) || pd.load_param(text))
         return -1;
     if (pd.get(0, 0) != INT_MIN || pd.get(1, 0) != INT_MAX
@@ -884,8 +884,35 @@ static int test_paramdict_text_boundaries()
         }
     }
 
-    // exercise array growth and the final copy for both element types
-    const int array_lengths[] = {16, 17, 32, 33, 257};
+    // lines may end at a read-buffer boundary or split a quoted/numeric token across it
+    const int line_lengths[] = {1022, 1023, 1024, 1035, 1048, 2046, 2047};
+    for (size_t j = 0; j < sizeof(line_lengths) / sizeof(line_lengths[0]); j++)
+        for (int leading = 0; leading < 2; leading++)
+        {
+            const char* params = "0=\"hello world\" 1=1.25,2.5 2=42";
+            const std::string padding = make_param_string(line_lengths[j] - strlen(params), ' ');
+            std::string input = leading ? padding : std::string(params);
+            input += leading ? std::string(params) : padding;
+            input += "\r\n+Probe next 1 1 in out\n";
+            const unsigned char* ptr = (const unsigned char*)input.c_str();
+            ncnn::DataReaderFromMemory dr(ptr);
+            if (pd.load_param(dr) || pd.get(0, std::string()) != "hello world" || pd.get(2, 0) != 42)
+            {
+                fprintf(stderr, "ParamDict long line failed len=%d leading=%d\n", line_lengths[j], leading);
+                return -1;
+            }
+            const ncnn::Mat values = pd.get(1, ncnn::Mat());
+            char header[64];
+            if (values.w != 2 || values[0] != 1.25f || values[1] != 2.5f
+                    || dr.scan(" %63[^\r\n]", header) != 1 || strcmp(header, "+Probe next 1 1 in out"))
+            {
+                fprintf(stderr, "ParamDict long line array or next header failed\n");
+                return -1;
+            }
+        }
+
+    // exercise stack storage, array growth and the final copy for both element types
+    const int array_lengths[] = {1, 15, 16, 17, 31, 32, 33, 257, 1024};
     for (size_t j = 0; j < sizeof(array_lengths) / sizeof(array_lengths[0]); j++)
         for (int floating = 0; floating < 2; floating++)
         {
@@ -896,6 +923,8 @@ static int test_paramdict_text_boundaries()
                 snprintf(value, sizeof(value), "%s%d%s", i ? "," : "", i, floating ? ".0" : "");
                 input += value;
             }
+            if (array_lengths[j] == 1)
+                input += ",";
             input += " 1=42";
             if (check_text_result(input.c_str(), true) || pd.load_param(input.c_str()) || pd.get(1, 0) != 42)
             {
@@ -952,7 +981,7 @@ static int check_float_boundary(const char* text, float expected, int count, boo
 
 static int test_paramdict_float_boundaries()
 {
-    // combining the comma scan must retain the full numeric token limit
+    // old array elements retain the full numeric token limit
     for (int len = 127; len <= 128; len++)
     {
         std::string text = "-23300=1,0.";
@@ -1072,9 +1101,9 @@ static int check_paramdict_reload(const ncnn::DataReader& dr, bool binary)
         return -1;
     }
 
-    // leave each following layer header for the caller, including after a failed scan
-    char header[64];
-    if (!binary && (dr.scan(" %63[^\r\n]", header) != 1 || strcmp(header, "ReLU next 1 1 in out")))
+    // read each following header without consuming its parameters on the same line
+    char header[64] = {0};
+    if (!binary && (dr.scan(" %20c", header) != 1 || strcmp(header, "ReLU next 1 1 in out")))
     {
         fprintf(stderr, "ParamDict next layer header failed\n");
         return -1;
@@ -1086,7 +1115,7 @@ static int check_paramdict_reload(const ncnn::DataReader& dr, bool binary)
         return -1;
     }
 
-    if (!binary && (dr.scan(" %63[^\r\n]", header) != 1 || strcmp(header, "Clip last 1 1 out final")))
+    if (!binary && (dr.scan(" %23c", header) != 1 || strcmp(header, "Clip last 1 1 out final")))
     {
         fprintf(stderr, "ParamDict last layer header failed\n");
         return -1;
@@ -1107,8 +1136,8 @@ static int check_paramdict_reload(const ncnn::DataReader& dr, bool binary)
 
 static int test_paramdict_reload()
 {
-    // a literal '-' in the id scanset must not consume punctuation in layer types
-    const char* headers[] = {".Custom next 1 1 in out", "/Custom next 1 1 in out", ",Custom next 1 1 in out"};
+    // the next layer name is independent of parameter syntax, even with no parameters
+    const char* headers[] = {".Custom next 1 1 in out", "/Custom next 1 1 in out", ",Custom next 1 1 in out", "+Probe next 1 1 in out", "-Probe next 1 1 in out", "123Probe next 1 1 in out"};
     for (size_t i = 0; i < sizeof(headers) / sizeof(headers[0]); i++)
         for (int empty = 0; empty < 2; empty++)
         {
@@ -1136,8 +1165,8 @@ static int test_paramdict_reload()
             }
         }
 
-    const char* text = "0=42 1=1.5 2=3,4 3=hello 31=31\r\nReLU next 1 1 in out\r\n4=7\r\nClip last 1 1 out final\r\n";
-    const char* old_array_text = "0=42 1=1.5 -23302=2,3,4 3=hello 31=31\r\nReLU next 1 1 in out\r\n4=7\r\nClip last 1 1 out final\r\n";
+    const char* text = "0=42 1=1.5 2=3,4 3=hello 31=31\r\nReLU next 1 1 in out 4=7\r\nClip last 1 1 out final\r\n";
+    const char* old_array_text = "0=42 1=1.5 -23302=2,3,4 3=hello 31=31\r\nReLU next 1 1 in out 4=7\r\nClip last 1 1 out final\r\n";
 
     // three consecutive parameter blocks, with no parameters in the last block
     const int words[] = {0, 42, 1, 0x3fc00000, -23302, 2, 3, 4, -23403, 5, 0x6c6c6568, 0x6f, 31, 31, -233, 4, 7, -233, -233};