#include "load_metadata.h"
#include "repomd.h"
#include "compression_wrapper.h"
+#include "misc.h"
#define VERSION "0.1"
#define DEFAULT_WORKERS 5
#define DEFAULT_UNIQUE_MD_FILENAMES TRUE
+#define BUF_SIZE 2048
+
GRegex *location_subs_re; // Evil global variable
// Items filled by cmd option parser
+ char *input_dir; //
char *location_base; // Base URL location for all files
char *outputdir; // Output directory
char **excludes; // List of file globs to exclude
char *pkglist; // File with files to include
char **includepkg; // List of files to include
+ char *groupfile; // Groupfile
gboolean quiet; // Shut up!
gboolean verbose; // Verbosely more than usual
gboolean update; // Update repo if metadata already exists
// Items filled by check_arguments()
+ char *groupfile_fullpath;
GSList *exclude_masks;
GSList *include_pkgs;
GSList *l_update_md_paths;
{ "baseurl", 'u', 0, G_OPTION_ARG_FILENAME, &(cmd_options.location_base),
"Optional base URL location for all files.", "<URL>" },
{ "outputdir", 'o', 0, G_OPTION_ARG_FILENAME, &(cmd_options.outputdir),
- "Optional output directory", "<URL>" },
+ "Optional output directory.", "<URL>" },
{ "excludes", 'x', 0, G_OPTION_ARG_FILENAME_ARRAY, &(cmd_options.excludes),
"File globs to exclude, can be specified multiple times.", "<packages>" },
{ "pkglist", 'i', 0, G_OPTION_ARG_FILENAME, &(cmd_options.pkglist),
- "specify a text file which contains the complete list of files to include"
+ "Specify a text file which contains the complete list of files to include"
" in the repository from the set found in the directory. File format is"
" one package per line, no wildcards or globs.", "<filename>" },
{ "includepkg", 'n', 0, G_OPTION_ARG_FILENAME_ARRAY, &(cmd_options.includepkg),
- "specify pkgs to include on the command line. Takes urls as well as local paths.",
+ "Specify pkgs to include on the command line. Takes urls as well as local paths.",
"<packages>" },
+ { "groupfile", 'g', 0, G_OPTION_ARG_FILENAME, &(cmd_options.groupfile),
+ "Path to groupfile to include in metadata.",
+ "GROUPFILE" },
{ "quiet", 'q', 0, G_OPTION_ARG_NONE, &(cmd_options.quiet),
"Run quietly.", NULL },
{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &(cmd_options.verbose),
"recalculating it. In the case of a large repository with only a few new or modified rpms"
"this can significantly reduce I/O and processing time.", NULL },
{ "update-md-path", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &(cmd_options.update_md_paths),
- "Use the existing repodata for --update from this path", NULL },
+ "Use the existing repodata for --update from this path.", NULL },
{ "skip-stat", 0, 0, G_OPTION_ARG_NONE, &(cmd_options.skip_stat),
- "skip the stat() call on a --update, assumes if the filename is the same then the file is"
+ "Skip the stat() call on a --update, assumes if the filename is the same then the file is"
"still the same (only use this if you're fairly trusting or gullible).", NULL },
{ "version", 'V', 0, G_OPTION_ARG_NONE, &(cmd_options.version),
"Output version.", NULL},
{ "no-database", 0, 0, G_OPTION_ARG_NONE, &(cmd_options.no_database),
"Do not generate sqlite databases in the repository.", NULL },
{ "skip-symlinks", 'S', 0, G_OPTION_ARG_NONE, &(cmd_options.skip_symlinks),
- "Ignore symlinks of packages", NULL},
+ "Ignore symlinks of packages.", NULL},
{ "checksum", 's', 0, G_OPTION_ARG_STRING, &(cmd_options.checksum),
"Choose the checksum type used in repomd.xml and for packages in the metadata."
"The default is now \"sha256\".", "<checksum_type>" },
"Only import the last N changelog entries, from each rpm, into the metadata.",
"<number>" },
{ "unique-md-filenames", 0, 0, G_OPTION_ARG_NONE, &(cmd_options.unique_md_filenames),
- "Include the file's checksum in the metadata filename, helps HTTP caching (default)",
+ "Include the file's checksum in the metadata filename, helps HTTP caching (default).",
NULL },
{ "simple-md-filenames", 0, 0, G_OPTION_ARG_NONE, &(cmd_options.simple_md_filenames),
"Do not include the file's checksum in the metadata filename.", NULL },
{ "workers", 0, 0, G_OPTION_ARG_INT, &(cmd_options.workers),
- "number of workers to spawn to read rpms.", NULL },
+ "Number of workers to spawn to read rpms.", NULL },
{ NULL }
};
x++;
}
+ // Check groupfile
+ options->groupfile_fullpath = NULL;
+ if (options->groupfile) {
+ if (g_str_has_prefix(options->groupfile, "/")) {
+ options->groupfile_fullpath = g_strdup(options->groupfile);
+ } else {
+ options->groupfile_fullpath = g_strconcat(options->input_dir, options->groupfile, NULL);
+ }
+
+ if (!g_file_test(options->groupfile_fullpath, G_FILE_TEST_IS_REGULAR|G_FILE_TEST_EXISTS)) {
+ g_warning("groupfile %s doesn't exists", options->groupfile_fullpath);
+ return FALSE;
+ }
+ }
+
// Process pkglist file
if (options->pkglist) {
if (!g_file_test(options->pkglist, G_FILE_TEST_IS_REGULAR|G_FILE_TEST_EXISTS)) {
void free_options(struct CmdOptions *options)
{
+ g_free(options->input_dir);
g_free(options->location_base);
g_free(options->outputdir);
g_free(options->pkglist);
g_free(options->checksum);
+ g_free(options->groupfile);
+ g_free(options->groupfile_fullpath);
// Free excludes string list
int x = 0;
// ---------- DEBUG STUFF END */
- // Check parsed arguments
-
- if (!check_arguments(&cmd_options)) {
- free_options(&cmd_options);
- g_option_context_free(context);
- exit(1);
- }
-
- g_option_context_free(context);
+ // Arguments pre-check
if (cmd_options.version) {
puts("Version: "VERSION);
}
+ // Dirs
+
+ gchar *in_dir = NULL; // path/to/repo/
+ gchar *in_repo = NULL; // path/to/repo/repodata/
+ gchar *out_dir = NULL; // path/to/out_repo/
+ gchar *out_repo = NULL; // path/to/out_repo/repodata/
+ gchar *tmp_out_repo = NULL; // path/to/out_repo/.repodata/
+
+
+ // Normalize in_dir format (result has exactly only one traling '/')
+
+ int i = strlen(argv[1]);
+ do {
+ i--;
+ } while (argv[1][i] == '/');
+ in_dir = g_strndup(argv[1], i+2);
+ if (in_dir[i+1] != '/') {
+ in_dir[i+1] = '/';
+ }
+
+ cmd_options.input_dir = g_strdup(in_dir);
+
+
+ // Check parsed arguments
+
+ if (!check_arguments(&cmd_options)) {
+ g_free(in_dir);
+ free_options(&cmd_options);
+ g_option_context_free(context);
+ exit(1);
+ }
+
+ g_option_context_free(context);
+
+
// Set logging stuff
if (cmd_options.quiet) {
// Set paths of input and output repos
- gchar *in_dir = NULL; // path/to/repo/
- gchar *in_repo = NULL; // path/to/repo/repodata/
- gchar *out_dir = NULL; // path/to/out_repo/
- gchar *out_repo = NULL; // path/to/out_repo/repodata/
- gchar *tmp_out_repo = NULL; // path/to/out_repo/.repodata/
-
- // Normalize in_dir format (result has exactly only one traling '/')
- int i = strlen(argv[1]);
- do {
- i--;
- } while (argv[1][i] == '/');
- in_dir = g_strndup(argv[1], i+2);
- if (in_dir[i+1] != '/') {
- in_dir[i+1] = '/';
- }
-
in_repo = g_strconcat(in_dir, "repodata/", NULL);
if (cmd_options.outputdir) {
*/
+ // Copy groupfile
+ gchar *groupfile = NULL;
+ if (cmd_options.groupfile_fullpath) {
+ groupfile = g_strconcat(tmp_out_repo, get_filename(cmd_options.groupfile_fullpath), NULL);
+ g_debug("Copy groupfile %s -> %s", cmd_options.groupfile_fullpath, groupfile);
+ if (copy_file(cmd_options.groupfile_fullpath, groupfile) != CR_COPY_OK) {
+ g_critical("Error while copy %s -> %s", cmd_options.groupfile_fullpath, groupfile);
+ }
+ }
+
+
// Load old metadata if --update
GHashTable *old_metadata = NULL;
gchar *pri_xml_name = g_strconcat("repodata/", "primary.xml.gz", NULL);
gchar *fil_xml_name = g_strconcat("repodata/", "filelists.xml.gz", NULL);
gchar *oth_xml_name = g_strconcat("repodata/", "other.xml.gz", NULL);
+ gchar *groupfile_name = g_strconcat("repodata/", get_filename(groupfile), NULL);
- struct repomdResult *repomd_res = xml_repomd(out_dir, cmd_options.unique_md_filenames, pri_xml_name, fil_xml_name, oth_xml_name, NULL, NULL, NULL, &cmd_options.checksum_type);
+ struct repomdResult *repomd_res = xml_repomd(out_dir, cmd_options.unique_md_filenames, pri_xml_name, fil_xml_name, oth_xml_name, NULL, NULL, NULL, groupfile_name, &cmd_options.checksum_type);
gchar *repomd_path = g_strconcat(out_repo, "repomd.xml", NULL);
FILE *frepomd = fopen(repomd_path, "w");
g_free(pri_xml_name);
g_free(fil_xml_name);
g_free(oth_xml_name);
+ g_free(groupfile_name);
// Clean up
g_free(pri_xml_filename);
g_free(fil_xml_filename);
g_free(oth_xml_filename);
+ g_free(groupfile);
free_options(&cmd_options);
free_package_parser();
#define REPOMD_ERR 1
+struct repomdData {
+ const char *location_href;
+ char *checksum;
+ char *checksum_type;
+ char *checksum_open;
+ char *checksum_open_type;
+ long timestamp;
+ long size;
+ long size_open;
+ int db_ver;
+
+ GStringChunk *chunk;
+};
+
+
typedef struct _contentStat {
char *checksum;
long size;
} contentStat;
+struct repomdData *new_repomddata();
+void free_repomddata(struct repomdData *);
+void free_repomdresult(struct repomdResult *);
+
+
struct repomdData *new_repomddata()
{
struct repomdData *md = (struct repomdData *) g_malloc0(sizeof(struct repomdData));
g_free(rr->pri_sqlite_location);
g_free(rr->fil_sqlite_location);
g_free(rr->oth_sqlite_location);
+ g_free(rr->groupfile_location);
+ g_free(rr->cgroupfile_location);
g_free(rr->repomd_xml);
g_free(rr);
}
+void process_groupfile(const char *base_path, struct repomdData *groupfile,
+ struct repomdData *cgroupfile, ChecksumType *checksum_type)
+{
+ if (!groupfile || !(groupfile->location_href) || !strlen(groupfile->location_href)) {
+ return;
+ }
+
+
+ // Checksum stuff
+
+ const char *checksum_str = DEFAULT_CHECKSUM;
+ ChecksumType checksum_t = DEFAULT_CHECKSUM_ENUM_VAL;
+
+ if (checksum_type) {
+ checksum_str = get_checksum_name_str(*checksum_type);
+ checksum_t = *checksum_type;
+ }
+
+
+ // Paths
+
+ gchar *clocation_href = g_strconcat(groupfile->location_href, ".gz", NULL);
+ cgroupfile->location_href = g_string_chunk_insert(cgroupfile->chunk, clocation_href);
+ g_free(clocation_href);
+
+ gchar *path = g_strconcat(base_path, "/", groupfile->location_href, NULL);
+ gchar *cpath = g_strconcat(base_path, "/", cgroupfile->location_href, NULL);
+
+ if (!g_file_test(path, G_FILE_TEST_EXISTS|G_FILE_TEST_IS_REGULAR)) {
+ // File doesn't exists
+ g_warning(MODULE"process_groupfile: File %s doesn't exists", path);
+ return;
+ }
+
+
+ // Compress file + get size of non compressed file
+
+ int readed;
+ char buf[BUFFER_SIZE];
+ CW_FILE *cw_plain;
+ CW_FILE *cw_compressed;
+
+ cw_plain = cw_open(path, CW_MODE_READ, NO_COMPRESSION);
+ cw_compressed = cw_open(cpath, CW_MODE_WRITE, GZ_COMPRESSION);
+
+ while ((readed = cw_read(cw_plain, buf, BUFFER_SIZE)) > 0) {
+ if (cw_write(cw_compressed, buf, (unsigned int) readed) == CW_ERR) {
+ g_debug(MODULE"process_groupfile: Error while groupfile compression");
+ break;
+ }
+ }
+
+ cw_close(cw_compressed);
+ cw_close(cw_plain);
+
+ if (readed == CW_ERR) {
+ g_debug(MODULE"process_groupfile: Error while groupfile compression");
+ }
+
+
+ // Compute checksums
+
+ gchar *checksum;
+ gchar *cchecksum;
+ checksum = compute_file_checksum(path, checksum_t);
+ cchecksum = compute_file_checksum(cpath, checksum_t);
+
+
+ // Get stats
+
+ long gf_size = -1, cgf_size = -1;
+ long gf_time = -1, cgf_time = -1;
+ struct stat gf_stat, cgf_stat;
+
+ if (stat(path, &gf_stat)) {
+ g_debug(MODULE"process_groupfile: Error while stat() on %s", path);
+ } else {
+ gf_size = gf_stat.st_size;
+ gf_time = gf_stat.st_mtime;
+ }
+
+ if (stat(cpath, &cgf_stat)) {
+ g_debug(MODULE"process_groupfile: Error while stat() on %s", path);
+ } else {
+ cgf_size = cgf_stat.st_size;
+ cgf_time = cgf_stat.st_mtime;
+ }
+
+
+ // Results
+
+ groupfile->checksum = g_string_chunk_insert(groupfile->chunk, checksum);
+ groupfile->checksum_type = g_string_chunk_insert(groupfile->chunk, checksum_str);
+ groupfile->checksum_open = NULL;
+ groupfile->checksum_open_type = NULL;
+ groupfile->timestamp = gf_time;
+ groupfile->size = gf_size;
+ groupfile->size_open = -1;
+
+ cgroupfile->checksum = g_string_chunk_insert(cgroupfile->chunk, cchecksum);
+ cgroupfile->checksum_type = g_string_chunk_insert(cgroupfile->chunk, checksum_str);
+ cgroupfile->checksum_open = g_string_chunk_insert(groupfile->chunk, checksum);
+ cgroupfile->checksum_open_type = g_string_chunk_insert(groupfile->chunk, checksum_str);
+ cgroupfile->timestamp = cgf_time;
+ cgroupfile->size = cgf_size;
+ cgroupfile->size_open = gf_size;
+
+ g_free(checksum);
+ g_free(cchecksum);
+ g_free(path);
+ g_free(cpath);
+}
+
void dump_data_items(xmlTextWriterPtr writer, struct repomdData *md, const xmlChar *type)
{
xmlTextWriterWriteString(writer, BAD_CAST md->checksum);
xmlTextWriterEndElement(writer);
- xmlTextWriterStartElement(writer, BAD_CAST "open-checksum");
- xmlTextWriterWriteAttribute(writer, BAD_CAST "type", (xmlChar *) md->checksum_open_type);
- xmlTextWriterWriteString(writer, BAD_CAST md->checksum_open);
- xmlTextWriterEndElement(writer);
+ if (md->checksum_open) {
+ xmlTextWriterStartElement(writer, BAD_CAST "open-checksum");
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", (xmlChar *) md->checksum_open_type);
+ xmlTextWriterWriteString(writer, BAD_CAST md->checksum_open);
+ xmlTextWriterEndElement(writer);
+ }
xmlTextWriterStartElement(writer, BAD_CAST "location");
xmlTextWriterWriteAttribute(writer, BAD_CAST "href", (xmlChar *) md->location_href);
xmlTextWriterWriteFormatString(writer, "%ld", md->size);
xmlTextWriterEndElement(writer);
- xmlTextWriterStartElement(writer, BAD_CAST "open-size");
- xmlTextWriterWriteFormatString(writer, "%ld", md->size_open);
- xmlTextWriterEndElement(writer);
+ if (md->size_open != -1) {
+ xmlTextWriterStartElement(writer, BAD_CAST "open-size");
+ xmlTextWriterWriteFormatString(writer, "%ld", md->size_open);
+ xmlTextWriterEndElement(writer);
+ }
if (g_str_has_suffix((char *)type, "_db")) {
xmlTextWriterStartElement(writer, BAD_CAST "database_version");
char *repomd_xml_dump(long revision, struct repomdData *pri_xml, struct repomdData *fil_xml, struct repomdData *oth_xml,
- struct repomdData *pri_sqlite, struct repomdData *fil_sqlite, struct repomdData *oth_sqlite)
+ struct repomdData *pri_sqlite, struct repomdData *fil_sqlite, struct repomdData *oth_sqlite,
+ struct repomdData *groupfile, struct repomdData *cgroupfile)
{
xmlBufferPtr buf = xmlBufferCreate();
if (!buf) {
dump_data_items(writer, pri_sqlite, (const xmlChar *) "primary_db");
dump_data_items(writer, fil_sqlite, (const xmlChar *) "filelists_db");
dump_data_items(writer, oth_sqlite, (const xmlChar *) "other_db");
+ dump_data_items(writer, groupfile, (const xmlChar *) "group");
+ dump_data_items(writer, cgroupfile, (const xmlChar *) "group_gz");
xmlTextWriterEndElement(writer); // repomd element end
}
-
-struct repomdResult *xml_repomd_2(const char *path, int rename_to_unique, struct repomdData *pri_xml, struct repomdData *fil_xml, struct repomdData *oth_xml,
- struct repomdData *pri_sqlite, struct repomdData *fil_sqlite, struct repomdData *oth_sqlite, ChecksumType *checksum_type)
+// groupfile is expected uncompressed!
+struct repomdResult *xml_repomd_2(const char *path, int rename_to_unique,
+ struct repomdData *pri_xml, struct repomdData *fil_xml, struct repomdData *oth_xml,
+ struct repomdData *pri_sqlite, struct repomdData *fil_sqlite, struct repomdData *oth_sqlite,
+ struct repomdData *groupfile, struct repomdData *cgroupfile, ChecksumType *checksum_type)
{
if (!path) {
return NULL;
fill_missing_data(path, fil_sqlite, checksum_type);
fill_missing_data(path, oth_sqlite, checksum_type);
+ process_groupfile(path, groupfile, cgroupfile, checksum_type);
+
// Include checksum in the metadata filename
rename_file(path, pri_sqlite);
rename_file(path, fil_sqlite);
rename_file(path, oth_sqlite);
+ rename_file(path, groupfile);
+ rename_file(path, cgroupfile);
}
res->pri_sqlite_location = pri_sqlite ? g_strdup(pri_sqlite->location_href) : NULL;
res->fil_sqlite_location = fil_sqlite ? g_strdup(fil_sqlite->location_href) : NULL;
res->oth_sqlite_location = oth_sqlite ? g_strdup(oth_sqlite->location_href) : NULL;
+ res->groupfile_location = groupfile ? g_strdup(groupfile->location_href) : NULL;
+ res->cgroupfile_location = cgroupfile ? (char *) cgroupfile->location_href : NULL;
// Get revision
// Dump xml
- res->repomd_xml = repomd_xml_dump(revision, pri_xml, fil_xml, oth_xml, pri_sqlite, fil_sqlite, oth_sqlite);
+ res->repomd_xml = repomd_xml_dump(revision, pri_xml, fil_xml, oth_xml, pri_sqlite, fil_sqlite, oth_sqlite, groupfile, cgroupfile);
return res;
}
struct repomdResult *xml_repomd(const char *path, int rename_to_unique, const char *pri_xml, const char *fil_xml, const char *oth_xml,
- const char *pri_sqlite, const char *fil_sqlite, const char *oth_sqlite, ChecksumType *checksum_type)
+ const char *pri_sqlite, const char *fil_sqlite, const char *oth_sqlite, const char *groupfile, ChecksumType *checksum_type)
{
if (!path) {
return NULL;
struct repomdData *pri_sqlite_rd = NULL;
struct repomdData *fil_sqlite_rd = NULL;
struct repomdData *oth_sqlite_rd = NULL;
+ struct repomdData *groupfile_rd = NULL;
+ struct repomdData *cgroupfile_rd = NULL;
if (pri_xml) {
pri_xml_rd = new_repomddata();
oth_sqlite_rd = new_repomddata();
oth_sqlite_rd->location_href = oth_sqlite;
}
+ if (groupfile) {
+ groupfile_rd = new_repomddata();
+ groupfile_rd->location_href = groupfile;
+ cgroupfile_rd = new_repomddata();
+ cgroupfile_rd->location_href = groupfile;
+ }
// Dump xml
- struct repomdResult *res = xml_repomd_2(path, rename_to_unique, pri_xml_rd, fil_xml_rd, oth_xml_rd, pri_sqlite_rd, fil_sqlite_rd, oth_sqlite_rd, checksum_type);
+ struct repomdResult *res = xml_repomd_2(path, rename_to_unique,
+ pri_xml_rd, fil_xml_rd, oth_xml_rd,
+ pri_sqlite_rd, fil_sqlite_rd, oth_sqlite_rd,
+ groupfile_rd, cgroupfile_rd, checksum_type);
free_repomddata(pri_xml_rd);
free_repomddata(fil_xml_rd);
free_repomddata(pri_sqlite_rd);
free_repomddata(fil_sqlite_rd);
free_repomddata(oth_sqlite_rd);
+ free_repomddata(groupfile_rd);
+ free_repomddata(cgroupfile_rd);
return res;
}
#include <libxml/xmlwriter.h>
#include "package.h"
-struct repomdData {
- const char *location_href;
- char *checksum;
- char *checksum_type;
- char *checksum_open;
- char *checksum_open_type;
- long timestamp;
- long size;
- long size_open;
- int db_ver;
-
- GStringChunk *chunk;
-};
-
struct repomdResult {
char *pri_xml_location;
char *pri_sqlite_location;
char *fil_sqlite_location;
char *oth_sqlite_location;
+ char *groupfile_location;
+ char *cgroupfile_location;
char *repomd_xml;
};
-struct repomdData *new_repomddata();
-void free_repomddata(struct repomdData *);
void free_repomdresult(struct repomdResult *);
+// all files except groupfile must be compressed (only group file should by plain noncompressed xml)
struct repomdResult *xml_repomd(const char *path, int rename_to_unique, const char *pri_xml, const char *fil_xml, const char *oth_xml,
- const char *pri_sqlite, const char *fil_sqlite, const char *oth_sqlite, ChecksumType *checksum_type);
+ const char *pri_sqlite, const char *fil_sqlite, const char *oth_sqlite, const char *groupfile, ChecksumType *checksum_type);
#endif /* __C_CREATEREPOLIB_REPOMD_H__ */
projects / services / createrepo_c.git / commitdiff