Skip to content

Commit 1cabd0f

Browse files
CISCCISC
authored
jinja : attribute support for join, map and sort (ggml-org#18883)
* support negative array index and default value * attribute support (int and str) for join, map and sort * add tests * update CODEOWNERS * improve fixme sorting comment
1 parent 1ffb42b commit 1cabd0f

4 files changed

Lines changed: 144 additions & 38 deletions

File tree

‎CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
/common/common.* @ggerganov
1616
/common/console.* @ggerganov
1717
/common/http.* @angt
18+
/common/jinja/ @ngxson @CISC @aldehir
1819
/common/llguidance.* @ggerganov
1920
/common/log.* @ggerganov
2021
/common/peg-parser.* @aldehir

‎common/jinja/value.cpp‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -776,19 +776,30 @@ const func_builtins & value_array_t::get_builtins() const {
776776
if (!is_val<value_array>(args.get_pos(0))) {
777777
throw raised_exception("join() first argument must be an array");
778778
}
779-
value val_delim = args.get_kwarg_or_pos("d", 1);
780-
value val_attribute = args.get_kwarg_or_pos("attribute", 2);
781-
if (!val_attribute->is_undefined()) {
782-
throw not_implemented_exception("array attribute join not implemented");
783-
}
779+
value val_delim = args.get_kwarg_or_pos("d", 1);
780+
value attribute = args.get_kwarg_or_pos("attribute", 2);
784781
const auto & arr = args.get_pos(0)->as_array();
785-
std::string delim = is_val<value_string>(val_delim) ? val_delim->as_string().str() : "";
782+
const bool attr_is_int = is_val<value_int>(attribute);
783+
if (!attribute->is_undefined() && !is_val<value_string>(attribute) && !attr_is_int) {
784+
throw raised_exception("join() attribute must be string or integer");
785+
}
786+
const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
787+
const std::string delim = val_delim->is_undefined() ? "" : val_delim->as_string().str();
788+
const std::string attr_name = attribute->is_undefined() ? "" : attribute->as_string().str();
786789
std::string result;
787790
for (size_t i = 0; i < arr.size(); ++i) {
788-
if (!is_val<value_string>(arr[i]) && !is_val<value_int>(arr[i]) && !is_val<value_float>(arr[i])) {
791+
value val_arr = arr[i];
792+
if (!attribute->is_undefined()) {
793+
if (attr_is_int && is_val<value_array>(val_arr)) {
794+
val_arr = val_arr->at(attr_int);
795+
} else if (!attr_is_int && !attr_name.empty() && is_val<value_object>(val_arr)) {
796+
val_arr = val_arr->at(attr_name);
797+
}
798+
}
799+
if (!is_val<value_string>(val_arr) && !is_val<value_int>(val_arr) && !is_val<value_float>(val_arr)) {
789800
throw raised_exception("join() can only join arrays of strings or numerics");
790801
}
791-
result += arr[i]->as_string().str();
802+
result += val_arr->as_string().str();
792803
if (i < arr.size() - 1) {
793804
result += delim;
794805
}
@@ -803,26 +814,30 @@ const func_builtins & value_array_t::get_builtins() const {
803814
}},
804815
{"tojson", tojson},
805816
{"map", [](const func_args & args) -> value {
806-
args.ensure_count(2, 3);
817+
args.ensure_count(2);
807818
if (!is_val<value_array>(args.get_pos(0))) {
808819
throw raised_exception("map: first argument must be an array");
809820
}
810-
value attribute = args.get_kwarg_or_pos("attribute", 1);
811-
if (is_val<value_int>(attribute)) {
812-
throw not_implemented_exception("map: integer attribute not implemented");
821+
if (!is_val<value_kwarg>(args.get_args().at(1))) {
822+
throw not_implemented_exception("map: filter-mapping not implemented");
813823
}
814-
if (!is_val<value_string>(attribute)) {
824+
value attribute = args.get_kwarg_or_pos("attribute", 1);
825+
const bool attr_is_int = is_val<value_int>(attribute);
826+
if (!is_val<value_string>(attribute) && !attr_is_int) {
815827
throw raised_exception("map: attribute must be string or integer");
816828
}
817-
std::string attr_name = attribute->as_string().str();
829+
const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
830+
const std::string attr_name = attribute->as_string().str();
818831
value default_val = args.get_kwarg("default", mk_val<value_undefined>());
819832
auto out = mk_val<value_array>();
820833
auto arr = args.get_pos(0)->as_array();
821834
for (const auto & item : arr) {
822-
if (!is_val<value_object>(item)) {
823-
throw raised_exception("map: item is not an object");
835+
value attr_val;
836+
if (attr_is_int) {
837+
attr_val = is_val<value_array>(item) ? item->at(attr_int, default_val) : default_val;
838+
} else {
839+
attr_val = is_val<value_object>(item) ? item->at(attr_name, default_val) : default_val;
824840
}
825-
value attr_val = item->at(attr_name, default_val);
826841
out->push_back(attr_val);
827842
}
828843
return out;
@@ -848,29 +863,35 @@ const func_builtins & value_array_t::get_builtins() const {
848863
return arr_editable->pop_at(index);
849864
}},
850865
{"sort", [](const func_args & args) -> value {
851-
args.ensure_count(1, 3);
866+
args.ensure_count(1, 4);
852867
if (!is_val<value_array>(args.get_pos(0))) {
853868
throw raised_exception("sort: first argument must be an array");
854869
}
855-
bool reverse = args.get_kwarg("reverse", mk_val<value_undefined>())->as_bool();
856-
value attribute = args.get_kwarg("attribute", mk_val<value_undefined>());
857-
std::string attr = attribute->is_undefined() ? "" : attribute->as_string().str();
870+
value val_reverse = args.get_kwarg_or_pos("reverse", 1);
871+
value val_case = args.get_kwarg_or_pos("case_sensitive", 2);
872+
value attribute = args.get_kwarg_or_pos("attribute", 3);
873+
// FIXME: sorting is currently always case sensitive
874+
//const bool case_sensitive = val_case->as_bool(); // undefined == false
875+
const bool reverse = val_reverse->as_bool(); // undefined == false
876+
const bool attr_is_int = is_val<value_int>(attribute);
877+
const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
878+
const std::string attr_name = attribute->is_undefined() ? "" : attribute->as_string().str();
858879
std::vector<value> arr = cast_val<value_array>(args.get_pos(0))->as_array(); // copy
859880
std::sort(arr.begin(), arr.end(),[&](const value & a, const value & b) {
860881
value val_a = a;
861882
value val_b = b;
862883
if (!attribute->is_undefined()) {
863-
if (!is_val<value_object>(a) || !is_val<value_object>(b)) {
864-
throw raised_exception("sort: items are not objects");
884+
if (attr_is_int && is_val<value_array>(a) && is_val<value_array>(b)) {
885+
val_a = a->at(attr_int);
886+
val_b = b->at(attr_int);
887+
} else if (!attr_is_int && !attr_name.empty() && is_val<value_object>(a) && is_val<value_object>(b)) {
888+
val_a = a->at(attr_name);
889+
val_b = b->at(attr_name);
890+
} else {
891+
throw raised_exception("sort: unsupported object attribute comparison");
865892
}
866-
val_a = attr.empty() ? a : a->at(attr);
867-
val_b = attr.empty() ? b : b->at(attr);
868-
}
869-
if (reverse) {
870-
return value_compare(val_a, val_b, value_compare_op::gt);
871-
} else {
872-
return !value_compare(val_a, val_b, value_compare_op::gt);
873893
}
894+
return value_compare(val_a, val_b, reverse ? value_compare_op::gt : value_compare_op::lt);
874895
});
875896
return mk_val<value_array>(arr);
876897
}},
@@ -964,7 +985,7 @@ const func_builtins & value_object_t::get_builtins() const {
964985
value val_case = args.get_kwarg_or_pos("case_sensitive", 1);
965986
value val_by = args.get_kwarg_or_pos("by", 2);
966987
value val_reverse = args.get_kwarg_or_pos("reverse", 3);
967-
// FIXME: sorting is case sensitive
988+
// FIXME: sorting is currently always case sensitive
968989
//const bool case_sensitive = val_case->as_bool(); // undefined == false
969990
const bool reverse = val_reverse->as_bool(); // undefined == false
970991
if (!val_by->is_undefined()) {

‎common/jinja/value.h‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,20 @@ struct value_t {
168168
}
169169
return val_obj.unordered.at(key);
170170
}
171-
virtual value & at(size_t index) {
172-
if (index >= val_arr.size()) {
171+
virtual value & at(int64_t index, value & default_val) {
172+
if (index < 0) {
173+
index += val_arr.size();
174+
}
175+
if (index < 0 || static_cast<size_t>(index) >= val_arr.size()) {
176+
return default_val;
177+
}
178+
return val_arr[index];
179+
}
180+
virtual value & at(int64_t index) {
181+
if (index < 0) {
182+
index += val_arr.size();
183+
}
184+
if (index < 0 || static_cast<size_t>(index) >= val_arr.size()) {
173185
throw std::runtime_error("Index " + std::to_string(index) + " out of bounds for array of size " + std::to_string(val_arr.size()));
174186
}
175187
return val_arr[index];

‎tests/test-jinja.cpp‎

Lines changed: 77 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,32 @@ static void test_filters(testing & t) {
389389
"123"
390390
);
391391

392+
test_template(t, "sort reverse",
393+
"{% for i in items|sort(true) %}{{ i }}{% endfor %}",
394+
{{"items", json::array({3, 1, 2})}},
395+
"321"
396+
);
397+
398+
test_template(t, "sort with attribute",
399+
"{{ items|sort(attribute='name')|join(attribute='age') }}",
400+
{{"items", json::array({
401+
json({{"name", "c"}, {"age", 3}}),
402+
json({{"name", "a"}, {"age", 1}}),
403+
json({{"name", "b"}, {"age", 2}}),
404+
})}},
405+
"123"
406+
);
407+
408+
test_template(t, "sort with numeric attribute",
409+
"{{ items|sort(attribute=0)|join(attribute=1) }}",
410+
{{"items", json::array({
411+
json::array({3, "z"}),
412+
json::array({1, "x"}),
413+
json::array({2, "y"}),
414+
})}},
415+
"xyz"
416+
);
417+
392418
test_template(t, "join",
393419
"{{ items|join(', ') }}",
394420
{{"items", json::array({"a", "b", "c"})}},
@@ -1000,7 +1026,17 @@ static void test_array_methods(testing & t) {
10001026
);
10011027

10021028
test_template(t, "array|join attribute",
1003-
"{{ arr|join(attribute=0) }}",
1029+
"{{ arr|join(attribute='age') }}",
1030+
{{"arr", json::array({
1031+
json({{"name", "a"}, {"age", 1}}),
1032+
json({{"name", "b"}, {"age", 2}}),
1033+
json({{"name", "c"}, {"age", 3}}),
1034+
})}},
1035+
"123"
1036+
);
1037+
1038+
test_template(t, "array|join numeric attribute",
1039+
"{{ arr|join(attribute=-1) }}",
10041040
{{"arr", json::array({json::array({1}), json::array({2}), json::array({3})})}},
10051041
"123"
10061042
);
@@ -1023,8 +1059,8 @@ static void test_array_methods(testing & t) {
10231059
"a,b,c,d"
10241060
);
10251061

1026-
test_template(t, "array.map() with attribute",
1027-
"{% for v in arr.map('age') %}{{ v }} {% endfor %}",
1062+
test_template(t, "array|map with attribute",
1063+
"{% for v in arr|map(attribute='age') %}{{ v }} {% endfor %}",
10281064
{{"arr", json::array({
10291065
json({{"name", "a"}, {"age", 1}}),
10301066
json({{"name", "b"}, {"age", 2}}),
@@ -1033,8 +1069,28 @@ static void test_array_methods(testing & t) {
10331069
"1 2 3 "
10341070
);
10351071

1036-
test_template(t, "array.map() with numeric attribute",
1037-
"{% for v in arr.map(0) %}{{ v }} {% endfor %}",
1072+
test_template(t, "array|map with attribute default",
1073+
"{% for v in arr|map(attribute='age', default=3) %}{{ v }} {% endfor %}",
1074+
{{"arr", json::array({
1075+
json({{"name", "a"}, {"age", 1}}),
1076+
json({{"name", "b"}, {"age", 2}}),
1077+
json({{"name", "c"}}),
1078+
})}},
1079+
"1 2 3 "
1080+
);
1081+
1082+
test_template(t, "array|map without attribute default",
1083+
"{% for v in arr|map(attribute='age') %}{{ v }} {% endfor %}",
1084+
{{"arr", json::array({
1085+
json({{"name", "a"}, {"age", 1}}),
1086+
json({{"name", "b"}, {"age", 2}}),
1087+
json({{"name", "c"}}),
1088+
})}},
1089+
"1 2 "
1090+
);
1091+
1092+
test_template(t, "array|map with numeric attribute",
1093+
"{% for v in arr|map(attribute=0) %}{{ v }} {% endfor %}",
10381094
{{"arr", json::array({
10391095
json::array({10, "x"}),
10401096
json::array({20, "y"}),
@@ -1043,6 +1099,22 @@ static void test_array_methods(testing & t) {
10431099
"10 20 30 "
10441100
);
10451101

1102+
test_template(t, "array|map with negative attribute",
1103+
"{% for v in arr|map(attribute=-1) %}{{ v }} {% endfor %}",
1104+
{{"arr", json::array({
1105+
json::array({10, "x"}),
1106+
json::array({20, "y"}),
1107+
json::array({30, "z"}),
1108+
})}},
1109+
"x y z "
1110+
);
1111+
1112+
test_template(t, "array|map with filter",
1113+
"{{ arr|map('int')|sum }}",
1114+
{{"arr", json::array({"1", "2", "3"})}},
1115+
"6"
1116+
);
1117+
10461118
// not used by any chat templates
10471119
// test_template(t, "array.insert()",
10481120
// "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}",

0 commit comments

Comments
 (0)