* Ecole Normale Superieure, 45 rue d’Ulm, 75230 Paris, France
*/
+#include <limits.h>
#include <isl/aff.h>
#include <isl/set.h>
#include <isl/ilp.h>
identity = isl_set_identity(isl_map_range(isl_map_copy(executed)));
executed = isl_map_domain_product(executed, identity);
+ build = isl_ast_build_set_single_valued(build, 1);
list = generate_code(isl_union_map_from_map(executed), build, 1);
* in generate_non_single_valued.
* Note that the inverse schedule being single-valued may depend
* on constraints that are only available in the original context
- * domain specified by the user. If the bare inverse schedule
- * is not single-valued, we double-check after introducing the constraints
- * from data->build->domain.
+ * domain specified by the user. We therefore first introduce
+ * the constraints from data->build->domain.
+ * On the other hand, we only perform the test after having taken the gist
+ * of the domain as the resulting map is the one from which the call
+ * expression is constructed. Using this map to construct the call
+ * expression usually yields simpler results.
+ * Because we perform the single-valuedness test on the gisted map,
+ * we may in rare cases fail to recognize that the inverse schedule
+ * is single-valued. This becomes problematic if this happens
+ * from the recursive call through generate_non_single_valued
+ * as we would then end up in an infinite recursion.
+ * We therefore check if we are inside a call to generate_non_single_valued
+ * and revert to the ungisted map if the gisted map turns out not to be
+ * single-valued.
*
* Otherwise, we generate a call expression for the single executed
* domain element and put a guard around it based on the (simplified)
isl_map *map;
int sv;
- sv = isl_map_is_single_valued(executed);
- if (sv < 0)
- goto error;
- if (!sv) {
- map = isl_map_copy(executed);
- map = isl_map_intersect_domain(map,
+ executed = isl_map_intersect_domain(executed,
isl_set_copy(data->build->domain));
- sv = isl_map_is_single_valued(map);
- isl_map_free(map);
- }
- if (!sv)
- return generate_non_single_valued(executed, data);
executed = isl_map_coalesce(executed);
map = isl_map_copy(executed);
map = isl_ast_build_compute_gist_map_domain(data->build, map);
+ sv = isl_map_is_single_valued(map);
+ if (sv < 0)
+ goto error;
+ if (!sv) {
+ isl_map_free(map);
+ if (data->build->single_valued)
+ map = isl_map_copy(executed);
+ else
+ return generate_non_single_valued(executed, data);
+ }
guard = isl_map_domain(isl_map_copy(map));
guard = isl_set_coalesce(guard);
guard = isl_ast_build_compute_gist(data->build, guard);
return 0;
error:
+ isl_map_free(map);
isl_map_free(executed);
return -1;
}
return data.list;
}
+/* Call the before_each_for callback, if requested by the user.
+ */
+static __isl_give isl_ast_node *before_each_for(__isl_take isl_ast_node *node,
+ __isl_keep isl_ast_build *build)
+{
+ isl_id *id;
+
+ if (!node || !build)
+ return isl_ast_node_free(node);
+ if (!build->before_each_for)
+ return node;
+ id = build->before_each_for(build, build->before_each_for_user);
+ node = isl_ast_node_set_annotation(node, id);
+ return node;
+}
+
+/* Call the after_each_for callback, if requested by the user.
+ */
+static __isl_give isl_ast_graft *after_each_for(__isl_keep isl_ast_graft *graft,
+ __isl_keep isl_ast_build *build)
+{
+ if (!graft || !build)
+ return isl_ast_graft_free(graft);
+ if (!build->after_each_for)
+ return graft;
+ graft->node = build->after_each_for(graft->node, build,
+ build->after_each_for_user);
+ if (!graft->node)
+ return isl_ast_graft_free(graft);
+ return graft;
+}
+
/* Eliminate the schedule dimension "pos" from "executed" and return
* the result.
*/
isl_ctx *ctx;
isl_ast_expr *inc;
+ if (!build)
+ return NULL;
ctx = isl_ast_build_get_ctx(build);
depth = isl_ast_build_get_depth(build);
* we performed separation with explicit bounds.
* The very first step is then to copy these constraints to "bounds".
*
+ * Since we may be calling before_each_for and after_each_for
+ * callbacks, we record the current inverse schedule in the build.
+ *
* We consider three builds,
* "build" is the one in which the current level is created,
* "body_build" is the build in which the next level is created,
domain = isl_set_detect_equalities(domain);
hull = isl_set_unshifted_simple_hull(isl_set_copy(domain));
bounds = isl_basic_set_intersect(bounds, hull);
+ build = isl_ast_build_set_executed(build, isl_union_map_copy(executed));
depth = isl_ast_build_get_depth(build);
sub_build = isl_ast_build_copy(build);
body_build = isl_ast_build_copy(sub_build);
body_build = isl_ast_build_increase_depth(body_build);
+ if (!eliminated)
+ node = before_each_for(node, body_build);
children = generate_next_level(executed,
isl_ast_build_copy(body_build));
- graft = isl_ast_graft_alloc_level(children, sub_build);
+ graft = isl_ast_graft_alloc_level(children, build, sub_build);
if (!eliminated)
graft = isl_ast_graft_insert_for(graft, node);
if (eliminated)
graft = refine_degenerate(graft, bounds, build, sub_build);
else
graft = refine_generic(graft, bounds, domain, build);
+ if (!eliminated)
+ graft = after_each_for(graft, body_build);
isl_ast_build_free(body_build);
isl_ast_build_free(sub_build);
if (isl_aff_get_denominator(offset, &data.d) < 0)
executed = isl_union_map_free(executed);
- if (isl_int_is_divisible_by(data.m, data.d))
+ if (executed && isl_int_is_divisible_by(data.m, data.d))
isl_int_divexact(data.m, data.m, data.d);
else
isl_int_set_si(data.m, 1);
data.depth = isl_ast_build_get_depth(build);
data.piece = domain_list->p;
g = isl_tarjan_graph_init(ctx, n, &domain_follows_at_depth, &data);
+ if (!g)
+ goto error;
i = 0;
while (list && n) {
* of domains that share some values of the outer dimensions.
* That is, domains in different components do not share any values
* of the outer dimensions. This means that these components
- * can be freely reorderd.
+ * can be freely reordered.
* Within each of the components, we sort the domains according
* to the execution order at the current depth.
*
if (n > 0)
list = isl_ast_graft_list_free(list);
- list = isl_ast_graft_list_sort(list);
+ list = isl_ast_graft_list_sort_guard(list);
isl_tarjan_graph_free(g);
return 0;
}
- if (!data->lower || isl_int_cmp_si(data->tmp, *data->n) < 0) {
+ if (isl_int_cmp_si(data->tmp, INT_MAX) <= 0 &&
+ (!data->lower || isl_int_cmp_si(data->tmp, *data->n) < 0)) {
isl_aff_free(data->lower);
data->lower = lower;
*data->n = isl_int_get_si(data->tmp);
return isl_aff_free(data.lower);
}
-/* Intersect "set" with the constraint
+/* Return the constraint
*
* i_"depth" = aff + offset
*/
-static __isl_give isl_set *at_offset(__isl_take isl_set *set, int depth,
- __isl_keep isl_aff *aff, int offset)
+static __isl_give isl_constraint *at_offset(int depth, __isl_keep isl_aff *aff,
+ int offset)
{
- isl_constraint *eq;
-
aff = isl_aff_copy(aff);
aff = isl_aff_add_coefficient_si(aff, isl_dim_in, depth, -1);
aff = isl_aff_add_constant_si(aff, offset);
- eq = isl_equality_from_aff(aff);
- set = isl_set_add_constraint(set, eq);
-
- return set;
+ return isl_equality_from_aff(aff);
}
/* Return a list of basic sets, one for each value of the current dimension
*
* We compute the unshifted simple hull of each slice to ensure that
* we have a single basic set per offset. The slicing constraint
- * is preserved by taking the unshifted simple hull, so these basic sets
+ * may get simplified away before the unshifted simple hull is taken
+ * and may therefore in some rare cases disappear from the result.
+ * We therefore explicitly add the constraint back after computing
+ * the unshifted simple hull to ensure that the basic sets
* remain disjoint. The constraints that are dropped by taking the hull
* will be taken into account at the next level, as in the case of the
* atomic option.
for (i = 0; list && i < n; ++i) {
isl_set *set;
isl_basic_set *bset;
+ isl_constraint *slice;
- set = at_offset(isl_set_copy(domain), depth, lower, i);
+ slice = at_offset(depth, lower, i);
+ set = isl_set_copy(domain);
+ set = isl_set_add_constraint(set, isl_constraint_copy(slice));
bset = isl_set_unshifted_simple_hull(set);
+ bset = isl_basic_set_add_constraint(bset, slice);
bset = isl_basic_set_apply(bset, isl_basic_map_copy(bmap));
list = isl_basic_set_list_add(list, bset);
}
* the user, except that inner dimensions have been eliminated and
* that they have been made pair-wise disjoint.
*
- * "includes_schedule_domain" is set if the "class_domain" (not stored
- * in this structure, but passed to the various functions) has been
- * intersected with "schedule_domain".
- *
* "sep_class" contains the user-specified split into separation classes
* specialized to the current depth.
- * "done" contains the union of th separation domains that have already
+ * "done" contains the union of the separation domains that have already
* been handled.
+ * "atomic" contains the domain that has effectively been made atomic.
+ * This domain may be larger than the intersection of option[atomic]
+ * and the schedule domain.
*/
struct isl_codegen_domains {
isl_basic_set_list *list;
isl_set *option[3];
- int includes_schedule_domain;
-
isl_map *sep_class;
isl_set *done;
+ isl_set *atomic;
};
/* Add domains to domains->list for each individual value of the current
/* Construct a single basic set that includes the intersection of
* the schedule domain, the atomic option domain and the class domain.
- * Add the resulting basic set to domains->list.
+ * Add the resulting basic set to domains->list and save a copy
+ * in domains->atomic for use in compute_partial_domains.
*
* We construct a single domain rather than trying to combine
* the schedule domains of individual domains because we are working
atomic_domain = isl_set_intersect(atomic_domain, isl_set_copy(domain));
empty = isl_set_is_empty(atomic_domain);
if (empty < 0 || empty) {
- isl_set_free(atomic_domain);
+ domains->atomic = atomic_domain;
return empty < 0 ? -1 : 0;
}
atomic_domain = isl_set_coalesce(atomic_domain);
bset = isl_set_unshifted_simple_hull(atomic_domain);
+ domains->atomic = isl_set_from_basic_set(isl_basic_set_copy(bset));
domains->list = isl_basic_set_list_add(domains->list, bset);
return 0;
* basic sets for which code should be generated separately
* for the given separation class domain.
*
+ * If any separation classes have been defined, then "class_domain"
+ * is the domain of the current class and does not refer to inner dimensions.
+ * Otherwise, "class_domain" is the universe domain.
+ *
* We first make sure that the class domain is disjoint from
* previously considered class domains.
*
*
* For atomic and remainder domains, inner dimensions and divs involving
* the current dimensions should be eliminated.
+ * In case we are working within a separation class, we need to intersect
+ * the result with the current "class_domain" to ensure that the domains
+ * are disjoint from those generated from other class domains.
+ *
+ * The domain that has been made atomic may be larger than specified
+ * by the user since it needs to be representable as a single basic set.
+ * This possibly larger domain is stored in domains->atomic by
+ * compute_atomic_domain.
*
* If anything is left after handling separate, unroll and atomic,
* we split it up into basic sets and append the basic sets to domains->list.
__isl_take isl_set *class_domain)
{
isl_basic_set_list *list;
+ isl_set *domain;
class_domain = isl_set_subtract(class_domain,
isl_set_copy(domains->done));
domains->done = isl_set_union(domains->done,
isl_set_copy(class_domain));
- if (compute_separate_domain(domains, class_domain) < 0)
+ domain = isl_set_copy(class_domain);
+
+ if (compute_separate_domain(domains, domain) < 0)
goto error;
- class_domain = isl_set_subtract(class_domain,
+ domain = isl_set_subtract(domain,
isl_set_copy(domains->option[separate]));
- if (!domains->includes_schedule_domain)
- class_domain = isl_set_intersect(class_domain,
- isl_set_copy(domains->schedule_domain));
+ domain = isl_set_intersect(domain,
+ isl_set_copy(domains->schedule_domain));
- if (compute_unroll_domains(domains, class_domain) < 0)
+ if (compute_unroll_domains(domains, domain) < 0)
goto error;
- class_domain = isl_set_subtract(class_domain,
+ domain = isl_set_subtract(domain,
isl_set_copy(domains->option[unroll]));
- class_domain = isl_ast_build_eliminate(domains->build,
- class_domain);
+ domain = isl_ast_build_eliminate(domains->build, domain);
+ domain = isl_set_intersect(domain, isl_set_copy(class_domain));
- if (compute_atomic_domain(domains, class_domain) < 0)
- goto error;
- class_domain = isl_set_subtract(class_domain,
- isl_set_copy(domains->option[atomic]));
+ if (compute_atomic_domain(domains, domain) < 0)
+ domain = isl_set_free(domain);
+ domain = isl_set_subtract(domain, domains->atomic);
- class_domain = isl_set_coalesce(class_domain);
- class_domain = isl_set_make_disjoint(class_domain);
+ domain = isl_set_coalesce(domain);
+ domain = isl_set_make_disjoint(domain);
- list = isl_basic_set_list_from_set(class_domain);
+ list = isl_basic_set_list_from_set(domain);
domains->list = isl_basic_set_list_concat(domains->list, list);
+ isl_set_free(class_domain);
+
return 0;
error:
+ isl_set_free(domain);
isl_set_free(class_domain);
return -1;
}
class_set = isl_set_from_point(pnt);
domain = isl_map_domain(isl_map_intersect_range(
isl_map_copy(domains->sep_class), class_set));
+ domain = isl_ast_build_compute_gist(domains->build, domain);
domain = isl_ast_build_eliminate(domains->build, domain);
disjoint = isl_set_plain_is_disjoint(domain, domains->schedule_domain);
return 0;
}
- domains->includes_schedule_domain = 0;
return compute_partial_domains(domains, domain);
}
* and split up the domain for each of them separately.
* Finally, we consider the remainder. If no separation classes were
* specified, then we call compute_partial_domains with the universe
- * "class_domain". Otherwise, we take the "schedule_domain" as "class_domain"
- * and set includes_schedule_domain to reflect that the schedule domain
- * has already been taken into account. We do this because we want to
+ * "class_domain". Otherwise, we take the "schedule_domain" as "class_domain",
+ * with inner dimensions removed. We do this because we want to
* avoid computing the complement of the class domains (i.e., the difference
* between the universe and domains->done).
*/
isl_space *space;
int n_param;
enum isl_ast_build_domain_type type;
+ int empty;
+
+ if (!executed)
+ return NULL;
ctx = isl_union_map_get_ctx(executed);
domains.list = isl_basic_set_list_alloc(ctx, 0);
domains.list = isl_basic_set_list_free(domains.list);
isl_set_free(classes);
- if (!domains.done)
+ empty = isl_set_is_empty(domains.done);
+ if (empty < 0) {
domains.list = isl_basic_set_list_free(domains.list);
- domains.includes_schedule_domain = !isl_set_is_empty(domains.done);
- if (!domains.includes_schedule_domain) {
+ domain = isl_set_free(domain);
+ } else if (empty) {
isl_set_free(domain);
domain = isl_set_universe(isl_set_get_space(domains.done));
+ } else {
+ domain = isl_ast_build_eliminate(build, domain);
}
if (compute_partial_domains(&domains, domain) < 0)
domains.list = isl_basic_set_list_free(domains.list);
isl_ast_node *node;
isl_union_map *executed;
+ build = isl_ast_build_copy(build);
+ build = isl_ast_build_set_single_valued(build, 0);
executed = isl_union_map_reverse(schedule);
list = generate_code(executed, isl_ast_build_copy(build), 0);
node = isl_ast_node_from_graft_list(list, build);
+ isl_ast_build_free(build);
return node;
}
projects / platform / upstream / isl.git / blobdiff