Skip to content

Commit e5fb547

Browse files
fangqfangq
committed
[jdata4] fix _ArrayChunk_ and _ArrayCoord_ definition in the updated draft-4
1 parent c2fc1dd commit e5fb547

3 files changed

Lines changed: 221 additions & 66 deletions

File tree

‎jdatadecode.m‎

Lines changed: 80 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -279,54 +279,94 @@
279279
end
280280
ndata = typecast(allbytes, arraytype);
281281
else
282-
% N-D chunked decode: iterate 1D row-major cell, fill tiles directly
283-
arrsize = double(data(j).(N_ArrayZipSize));
284-
ndim = numel(arrsize);
285-
nchunks_nd = ceil(arrsize ./ chunkshape);
286-
ntiles = prod(nchunks_nd);
287-
iscpx_arr = isfield(data, N_ArrayIsComplex) && data(j).(N_ArrayIsComplex);
288-
if (iscpx_arr)
289-
ndata = complex(cast(zeros(arrsize), arraytype));
290-
else
291-
ndata = cast(zeros(arrsize), arraytype);
292-
end
293-
tidx = cell(1, ndim);
294-
for ci = 1:ntiles
295-
[tidx{:}] = ind2sub(nchunks_nd, ci);
296-
chunkblob = uint8(chunks_cell{ci}(:)');
297-
if (needbase64 && ~strcmp(zipmethod, 'base64'))
298-
chunkblob = base64decode(chunkblob);
282+
% N-D chunked decode: two modes depending on array type.
283+
% Dense: _ArrayChunks_ is in _ArraySize_ (original) order;
284+
% tiles the row-major permuted intermediate.
285+
% Non-dense (sparse/shaped): _ArrayChunks_ is in _ArrayZipSize_ order;
286+
% tiles the logical processed array.
287+
isdense_chunk = ~(isfield(data, N_ArrayIsSparse) && data(j).(N_ArrayIsSparse)) && ~isfield(data, N_ArrayShape);
288+
if (isdense_chunk)
289+
origsize = double(data(j).(N_ArraySize)(:)'); % e.g. [3,4,5]
290+
ndim = numel(origsize);
291+
chunkshape_perm = fliplr(chunkshape); % permuted-space chunk
292+
arrsize_perm = fliplr(origsize); % e.g. [5,4,3]
293+
chunkshape_perm(end + 1:ndim) = arrsize_perm(numel(chunkshape_perm) + 1:end);
294+
chunkshape_perm = min(chunkshape_perm, arrsize_perm);
295+
nchunks_nd = ceil(arrsize_perm ./ chunkshape_perm);
296+
ntiles = prod(nchunks_nd);
297+
iscpx_arr = isfield(data, N_ArrayIsComplex) && data(j).(N_ArrayIsComplex);
298+
if (iscpx_arr)
299+
ndata = complex(cast(zeros(arrsize_perm), arraytype));
300+
else
301+
ndata = cast(zeros(arrsize_perm), arraytype);
299302
end
300-
rawchunk = typecast(decompfun(chunkblob, decodeparam{:}), arraytype);
301-
ranges = cell(1, ndim);
302-
tilesize = zeros(1, ndim);
303-
for d = 1:ndim
304-
r1 = (tidx{d} - 1) * chunkshape(d) + 1;
305-
r2 = min(tidx{d} * chunkshape(d), arrsize(d));
306-
ranges{d} = r1:r2;
307-
tilesize(d) = r2 - r1 + 1;
303+
tidx = cell(1, ndim);
304+
for ci = 1:ntiles
305+
[tidx{:}] = ind2sub(nchunks_nd, ci);
306+
chunkblob = uint8(chunks_cell{ci}(:)');
307+
if (needbase64 && ~strcmp(zipmethod, 'base64'))
308+
chunkblob = base64decode(chunkblob);
309+
end
310+
rawchunk = typecast(decompfun(chunkblob, decodeparam{:}), arraytype);
311+
ranges = cell(1, ndim);
312+
tilesize = zeros(1, ndim);
313+
for d = 1:ndim
314+
r1 = (tidx{d} - 1) * chunkshape_perm(d) + 1;
315+
r2 = min(tidx{d} * chunkshape_perm(d), arrsize_perm(d));
316+
ranges{d} = r1:r2;
317+
tilesize(d) = r2 - r1 + 1;
318+
end
319+
if (iscpx_arr)
320+
half = numel(rawchunk) / 2;
321+
ndata(ranges{:}) = complex(reshape(rawchunk(1:half), tilesize), ...
322+
reshape(rawchunk(half + 1:end), tilesize));
323+
else
324+
ndata(ranges{:}) = reshape(rawchunk, tilesize);
325+
end
308326
end
327+
% Flatten assembled permuted array to row for downstream processing
309328
if (iscpx_arr)
310-
half = numel(rawchunk) / 2;
311-
ndata(ranges{:}) = complex(reshape(rawchunk(1:half), tilesize), ...
312-
reshape(rawchunk(half + 1:end), tilesize));
329+
ndata = [real(ndata(:))', imag(ndata(:))'];
330+
data(j).(N_ArrayZipSize) = [2, numel(ndata) / 2];
313331
else
314-
ndata(ranges{:}) = reshape(rawchunk, tilesize);
332+
ndata = ndata(:).';
333+
data(j).(N_ArrayZipSize) = [1, numel(ndata)];
315334
end
316-
end
317-
if (iscpx_arr)
318-
ndata = [real(ndata(:))', imag(ndata(:))'];
335+
dims = data(j).(N_ArrayZipSize);
319336
else
320-
ndata = ndata(:).';
337+
% Non-dense: tile logical _ArrayZipSize_ shape; keep ZipSize unchanged.
338+
zipsize = double(data(j).(N_ArrayZipSize)(:)'); % e.g. [3,K] sparse
339+
ndim = numel(zipsize);
340+
chunkshape_nd = chunkshape;
341+
chunkshape_nd(end + 1:ndim) = zipsize(numel(chunkshape_nd) + 1:end);
342+
chunkshape_nd = min(chunkshape_nd, zipsize);
343+
nchunks_nd = ceil(zipsize ./ chunkshape_nd);
344+
ntiles = prod(nchunks_nd);
345+
ndata = cast(zeros(zipsize), arraytype);
346+
tidx = cell(1, ndim);
347+
for ci = 1:ntiles
348+
[tidx{:}] = ind2sub(nchunks_nd, ci);
349+
chunkblob = uint8(chunks_cell{ci}(:)');
350+
if (needbase64 && ~strcmp(zipmethod, 'base64'))
351+
chunkblob = base64decode(chunkblob);
352+
end
353+
rawchunk = typecast(decompfun(chunkblob, decodeparam{:}), arraytype);
354+
ranges = cell(1, ndim);
355+
tilesize = zeros(1, ndim);
356+
for d = 1:ndim
357+
r1 = (tidx{d} - 1) * chunkshape_nd(d) + 1;
358+
r2 = min(tidx{d} * chunkshape_nd(d), zipsize(d));
359+
ranges{d} = r1:r2;
360+
tilesize(d) = r2 - r1 + 1;
361+
end
362+
ndata(ranges{:}) = reshape(rawchunk, tilesize);
363+
end
364+
% Re-flatten to "rows-concatenated" format for downstream ZipSize block
365+
ndata = permute(ndata, ndim:-1:1);
366+
ndata = ndata(:)';
367+
% _ArrayZipSize_ stays unchanged; dims already set from line 236
321368
end
322369
end
323-
% Restore ZipSize for downstream complex reconstruction
324-
if (isfield(data, N_ArrayIsComplex) && data(j).(N_ArrayIsComplex))
325-
data(j).(N_ArrayZipSize) = [2, numel(ndata) / 2];
326-
else
327-
data(j).(N_ArrayZipSize) = [1, numel(ndata)];
328-
end
329-
dims = data(j).(N_ArrayZipSize);
330370
elseif (needbase64 && strcmp(zipmethod, 'base64') == 0)
331371
ndata = reshape(typecast(decompfun(base64decode(data(j).(N_ArrayZipData)), decodeparam{:}), arraytype), dims);
332372
else

‎jdataencode.m‎

Lines changed: 71 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -509,39 +509,85 @@
509509
end
510510
end
511511
newitem.(opt.N_ArrayChunks) = [chunklen];
512-
newitem.(opt.N_ArrayZipSize) = nelem;
512+
if (~isfield(newitem, opt.N_ArrayZipSize))
513+
newitem.(opt.N_ArrayZipSize) = nelem;
514+
end
513515
newitem.(opt.N_ArrayZipData) = zipchunks;
514516
else
515-
% N-D chunking: tile the permuted intermediate array by chunkshape.
516-
% 'item' here is the permuted ND intermediate (before flattening).
517-
arrsize = size(item);
518-
ndim = ndims(item);
519-
chunkshape(end + 1:ndim) = arrsize(numel(chunkshape) + 1:end); % pad to ndim
520-
chunkshape = min(chunkshape, arrsize);
521-
nchunks_nd = ceil(arrsize ./ chunkshape);
522-
zipchunks = cell(1, prod(nchunks_nd));
523-
tidx = cell(1, ndim);
524-
for ci = 1:prod(nchunks_nd)
525-
[tidx{:}] = ind2sub(nchunks_nd, ci);
526-
ranges = cell(1, ndim);
527-
for d = 1:ndim
528-
r1 = (tidx{d} - 1) * chunkshape(d) + 1;
529-
r2 = min(tidx{d} * chunkshape(d), arrsize(d));
530-
ranges{d} = r1:r2;
517+
% N-D chunking: two modes depending on array type.
518+
% Dense (real/complex): _ArrayChunks_ is in _ArraySize_ (original) order;
519+
% tiles the row-major permuted intermediate; _ArrayZipSize_ = [1/2, N_total].
520+
% Non-dense (sparse/shaped): _ArrayChunks_ is in _ArrayZipSize_ (processed) order;
521+
% tiles the processed _ArrayData_ matrix directly.
522+
isdense_chunk = ~(isfield(newitem, opt.N_ArrayIsSparse) && newitem.(opt.N_ArrayIsSparse)) && ~isfield(newitem, opt.N_ArrayShape);
523+
if (isdense_chunk)
524+
% Dense: item is already permuted to row-major.
525+
% chunkshape is in original _ArraySize_ order; flip to get permuted-space chunks.
526+
origsize = fliplr(size(item)); % original _ArraySize_ dimension order
527+
ndim = numel(origsize);
528+
chunkshape(end + 1:ndim) = origsize(numel(chunkshape) + 1:end);
529+
chunkshape = min(chunkshape, origsize); % clamp against original dims
530+
chunkshape_perm = fliplr(chunkshape); % chunk shape in permuted space
531+
arrsize_perm = size(item);
532+
nchunks_nd = ceil(arrsize_perm ./ chunkshape_perm);
533+
zipchunks = cell(1, prod(nchunks_nd));
534+
tidx = cell(1, ndim);
535+
for ci = 1:prod(nchunks_nd)
536+
[tidx{:}] = ind2sub(nchunks_nd, ci);
537+
ranges = cell(1, ndim);
538+
for d = 1:ndim
539+
r1 = (tidx{d} - 1) * chunkshape_perm(d) + 1;
540+
r2 = min(tidx{d} * chunkshape_perm(d), arrsize_perm(d));
541+
ranges{d} = r1:r2;
542+
end
543+
chunk = item(ranges{:});
544+
if (~isreal(item))
545+
chunk_flat = [real(chunk(:))', imag(chunk(:))'];
546+
zipchunks{ci} = compfun(typecast(chunk_flat, 'uint8'), encodeparam{:});
547+
else
548+
zipchunks{ci} = compfun(typecast(chunk(:)', 'uint8'), encodeparam{:});
549+
end
550+
if (opt.base64)
551+
zipchunks{ci} = char(base64encode(zipchunks{ci}));
552+
end
531553
end
532-
chunk = item(ranges{:});
554+
newitem.(opt.N_ArrayChunks) = chunkshape; % stored in original _ArraySize_ order
533555
if (~isreal(item))
534-
chunk_flat = [real(chunk(:))', imag(chunk(:))'];
535-
zipchunks{ci} = compfun(typecast(chunk_flat, 'uint8'), encodeparam{:});
556+
newitem.(opt.N_ArrayZipSize) = [2, numel(item)];
536557
else
537-
zipchunks{ci} = compfun(typecast(chunk(:)', 'uint8'), encodeparam{:});
558+
newitem.(opt.N_ArrayZipSize) = [1, numel(item)];
538559
end
539-
if (opt.base64)
540-
zipchunks{ci} = char(base64encode(zipchunks{ci}));
560+
else
561+
% Non-dense (sparse/shaped): tile the logical _ArrayZipSize_ shape.
562+
% _ArrayData_ is stored as a flat row; reshape to _ArrayZipSize_ for tiling.
563+
% _ArrayChunks_ is in _ArrayZipSize_ coordinate space.
564+
procsize = double(newitem.(opt.N_ArrayZipSize)); % e.g. [3,K] for sparse
565+
ndim = numel(procsize);
566+
chunkshape(end + 1:ndim) = procsize(numel(chunkshape) + 1:end);
567+
chunkshape = min(chunkshape, procsize);
568+
nchunks_nd = ceil(procsize ./ chunkshape);
569+
% Reshape flat _ArrayData_ row to logical procsize (inverse of rows-concatenated)
570+
flatdata = newitem.(opt.N_ArrayData)(:)';
571+
procdata = permute(reshape(flatdata, fliplr(procsize)), ndim:-1:1);
572+
zipchunks = cell(1, prod(nchunks_nd));
573+
tidx = cell(1, ndim);
574+
for ci = 1:prod(nchunks_nd)
575+
[tidx{:}] = ind2sub(nchunks_nd, ci);
576+
ranges = cell(1, ndim);
577+
for d = 1:ndim
578+
r1 = (tidx{d} - 1) * chunkshape(d) + 1;
579+
r2 = min(tidx{d} * chunkshape(d), procsize(d));
580+
ranges{d} = r1:r2;
581+
end
582+
chunk = procdata(ranges{:});
583+
zipchunks{ci} = compfun(typecast(chunk(:)', 'uint8'), encodeparam{:});
584+
if (opt.base64)
585+
zipchunks{ci} = char(base64encode(zipchunks{ci}));
586+
end
541587
end
588+
newitem.(opt.N_ArrayChunks) = chunkshape;
589+
% _ArrayZipSize_ was already set by the sparse/shaped encoding path; keep it.
542590
end
543-
newitem.(opt.N_ArrayChunks) = chunkshape;
544-
newitem.(opt.N_ArrayZipSize) = arrsize; % full intermediate shape
545591
newitem.(opt.N_ArrayZipData) = zipchunks;
546592
end
547593
newitem = rmfield(newitem, opt.N_ArrayData);

0 commit comments

Comments
 (0)