isl_basic_set_opt: avoid invalid access on error path

[platform/upstream/isl.git] / isl_equalities.c

diff --git a/isl_equalities.c b/isl_equalities.c
index 534900b..4447d9b 100644 (file)
--- a/isl_equalities.c
+++ b/isl_equalities.c
@@ -1,14 +1,14 @@
 /*
  * Copyright 2008-2009 Katholieke Universiteit Leuven
  *
- * Use of this software is governed by the GNU LGPLv2.1 license
+ * Use of this software is governed by the MIT license
  *
  * Written by Sven Verdoolaege, K.U.Leuven, Departement
  * Computerwetenschappen, Celestijnenlaan 200A, B-3001 Leuven, Belgium
  */
 
-#include "isl_mat.h"
-#include "isl_seq.h"
+#include <isl_mat_private.h>
+#include <isl/seq.h>
 #include "isl_map_private.h"
 #include "isl_equalities.h"
 
@@ -85,7 +85,7 @@ static struct isl_mat *particular_solution(struct isl_mat *B, struct isl_vec *d)
        M = isl_mat_left_hermite(M, 0, &U, NULL);
        if (!M || !U)
                goto error;
-       H = isl_mat_sub_alloc(B->ctx, M->row, 0, B->n_row, 0, B->n_row);
+       H = isl_mat_sub_alloc(M, 0, B->n_row, 0, B->n_row);
        H = isl_mat_lin_to_aff(H);
        C = isl_mat_inverse_product(H, C);
        if (!C)
@@ -98,8 +98,8 @@ static struct isl_mat *particular_solution(struct isl_mat *B, struct isl_vec *d)
        if (i < B->n_row)
                cst = isl_mat_alloc(B->ctx, B->n_row, 0);
        else
-               cst = isl_mat_sub_alloc(C->ctx, C->row, 1, B->n_row, 0, 1);
-       T = isl_mat_sub_alloc(U->ctx, U->row, B->n_row, B->n_col - 1, 0, B->n_row);
+               cst = isl_mat_sub_alloc(C, 1, B->n_row, 0, 1);
+       T = isl_mat_sub_alloc(U, B->n_row, B->n_col - 1, 0, B->n_row);
        cst = isl_mat_product(T, cst);
        isl_mat_free(M);
        isl_mat_free(C);
@@ -184,10 +184,14 @@ static struct isl_mat *parameter_compression_multi(
                                                D, U->row[j][k]);
        }
        A = isl_mat_left_hermite(A, 0, NULL, NULL);
-       T = isl_mat_sub_alloc(A->ctx, A->row, 0, A->n_row, 0, A->n_row);
+       T = isl_mat_sub_alloc(A, 0, A->n_row, 0, A->n_row);
        T = isl_mat_lin_to_aff(T);
+       if (!T)
+               goto error;
        isl_int_set(T->row[0][0], D);
        T = isl_mat_right_inverse(T);
+       if (!T)
+               goto error;
        isl_assert(T->ctx, isl_int_is_one(T->row[0][0]), goto error);
        T = isl_mat_transpose(T);
        isl_mat_free(A);
@@ -249,7 +253,7 @@ error:
  * then we divide this row of A by the common factor, unless gcd(A_i) = 0.
  * In the later case, we simply drop the row (in both A and d).
  *
- * If there are no rows left in A, the G is the identity matrix. Otherwise,
+ * If there are no rows left in A, then G is the identity matrix. Otherwise,
  * for each row i, we now determine the lattice of integer vectors
  * that satisfies this row.  Let U_i be the unimodular extension of the
  * row A_i.  This unimodular extension exists because gcd(A_i) = 1.
@@ -422,7 +426,7 @@ struct isl_mat *isl_mat_variable_compression(struct isl_mat *B,
                goto error;
 
        dim = B->n_col - 1;
-       H = isl_mat_sub_alloc(B->ctx, B->row, 0, B->n_row, 1, dim);
+       H = isl_mat_sub_alloc(B, 0, B->n_row, 1, dim);
        H = isl_mat_left_hermite(H, 0, &U, T2);
        if (!H || !U || (T2 && !*T2))
                goto error;
@@ -437,7 +441,7 @@ struct isl_mat *isl_mat_variable_compression(struct isl_mat *B,
                goto error;
        isl_int_set_si(C->row[0][0], 1);
        isl_mat_sub_neg(C->ctx, C->row+1, B->row, B->n_row, 0, 0, 1);
-       H1 = isl_mat_sub_alloc(H->ctx, H->row, 0, H->n_row, 0, H->n_row);
+       H1 = isl_mat_sub_alloc(H, 0, H->n_row, 0, H->n_row);
        H1 = isl_mat_lin_to_aff(H1);
        TC = isl_mat_inverse_product(H1, C);
        if (!TC)
@@ -460,10 +464,9 @@ struct isl_mat *isl_mat_variable_compression(struct isl_mat *B,
                }
                isl_int_set_si(TC->row[0][0], 1);
        }
-       U1 = isl_mat_sub_alloc(U->ctx, U->row, 0, U->n_row, 0, B->n_row);
+       U1 = isl_mat_sub_alloc(U, 0, U->n_row, 0, B->n_row);
        U1 = isl_mat_lin_to_aff(U1);
-       U2 = isl_mat_sub_alloc(U->ctx, U->row, 0, U->n_row,
-                               B->n_row, U->n_row - B->n_row);
+       U2 = isl_mat_sub_alloc(U, 0, U->n_row, B->n_row, U->n_row - B->n_row);
        U2 = isl_mat_lin_to_aff(U2);
        isl_mat_free(U);
        TC = isl_mat_product(U1, TC);
@@ -509,7 +512,7 @@ static struct isl_basic_set *compress_variables(
        if (bset->n_eq == 0)
                return bset;
 
-       B = isl_mat_sub_alloc(bset->ctx, bset->eq, 0, bset->n_eq, 0, 1 + dim);
+       B = isl_mat_sub_alloc6(bset->ctx, bset->eq, 0, bset->n_eq, 0, 1 + dim);
        TC = isl_mat_variable_compression(B, T2);
        if (!TC)
                goto error;
@@ -572,7 +575,7 @@ int isl_basic_set_dim_residue_class(struct isl_basic_set *bset,
        if (!bset || !modulo || !residue)
                return -1;
 
-       if (isl_basic_set_fast_dim_is_fixed(bset, pos, residue)) {
+       if (isl_basic_set_plain_dim_is_fixed(bset, pos, residue)) {
                isl_int_set_si(*modulo, 0);
                return 0;
        }
@@ -580,7 +583,7 @@ int isl_basic_set_dim_residue_class(struct isl_basic_set *bset,
        ctx = bset->ctx;
        total = isl_basic_set_total_dim(bset);
        nparam = isl_basic_set_n_param(bset);
-       H = isl_mat_sub_alloc(bset->ctx, bset->eq, 0, bset->n_eq, 1, total);
+       H = isl_mat_sub_alloc6(bset->ctx, bset->eq, 0, bset->n_eq, 1, total);
        H = isl_mat_left_hermite(H, 0, &U, NULL);
        if (!H)
                return -1;
@@ -601,11 +604,11 @@ int isl_basic_set_dim_residue_class(struct isl_basic_set *bset,
                goto error;
        isl_int_set_si(C->row[0][0], 1);
        isl_mat_sub_neg(C->ctx, C->row+1, bset->eq, bset->n_eq, 0, 0, 1);
-       H1 = isl_mat_sub_alloc(H->ctx, H->row, 0, H->n_row, 0, H->n_row);
+       H1 = isl_mat_sub_alloc(H, 0, H->n_row, 0, H->n_row);
        H1 = isl_mat_lin_to_aff(H1);
        C = isl_mat_inverse_product(H1, C);
        isl_mat_free(H);
-       U1 = isl_mat_sub_alloc(U->ctx, U->row, nparam+pos, 1, 0, bset->n_eq);
+       U1 = isl_mat_sub_alloc(U, nparam+pos, 1, 0, bset->n_eq);
        U1 = isl_mat_lin_to_aff(U1);
        isl_mat_free(U);
        C = isl_mat_product(U1, C);
@@ -667,20 +670,15 @@ int isl_set_dim_residue_class(struct isl_set *set,
        isl_int_init(r);
 
        for (i = 1; i < set->n; ++i) {
-               if (isl_basic_set_dim_residue_class(set->p[0], pos, &m, &r) < 0)
+               if (isl_basic_set_dim_residue_class(set->p[i], pos, &m, &r) < 0)
                        goto error;
                isl_int_gcd(*modulo, *modulo, m);
+               isl_int_sub(m, *residue, r);
+               isl_int_gcd(*modulo, *modulo, m);
                if (!isl_int_is_zero(*modulo))
                        isl_int_fdiv_r(*residue, *residue, *modulo);
                if (isl_int_is_one(*modulo))
                        break;
-               if (!isl_int_is_zero(*modulo))
-                       isl_int_fdiv_r(r, r, *modulo);
-               if (isl_int_ne(*residue, r)) {
-                       isl_int_set_si(*modulo, 1);
-                       isl_int_set_si(*residue, 0);
-                       break;
-               }
        }
 
        isl_int_clear(m);