From e992c3fe6d01d1b85932cfe7ba50e6c292841ee6 Mon Sep 17 00:00:00 2001 From: Sangwan Kwon Date: Mon, 27 Apr 2020 20:01:34 +0900 Subject: [PATCH] Reorganize directory structure Signed-off-by: Sangwan Kwon --- CMakeLists.txt | 21 +- {policy => doc/policy}/bluetooth.md | 0 {src => third-party}/rapidjson/allocators.h | 0 {src => third-party}/rapidjson/document.h | 0 .../rapidjson/encodedstream.h | 0 {src => third-party}/rapidjson/encodings.h | 0 {src => third-party}/rapidjson/error/en.h | 0 {src => third-party}/rapidjson/error/error.h | 0 .../rapidjson/filereadstream.h | 0 .../rapidjson/filewritestream.h | 0 {src => third-party}/rapidjson/fwd.h | 0 .../rapidjson/internal/biginteger.h | 0 .../rapidjson/internal/diyfp.h | 0 .../rapidjson/internal/dtoa.h | 0 .../rapidjson/internal/ieee754.h | 0 .../rapidjson/internal/itoa.h | 0 .../rapidjson/internal/meta.h | 0 .../rapidjson/internal/pow10.h | 0 .../rapidjson/internal/regex.h | 0 .../rapidjson/internal/stack.h | 0 .../rapidjson/internal/strfunc.h | 0 .../rapidjson/internal/strtod.h | 0 .../rapidjson/internal/swap.h | 0 .../rapidjson/istreamwrapper.h | 0 {src => third-party}/rapidjson/memorybuffer.h | 0 {src => third-party}/rapidjson/memorystream.h | 0 .../rapidjson/msinttypes/inttypes.h | 0 .../rapidjson/msinttypes/stdint.h | 0 .../rapidjson/ostreamwrapper.h | 0 {src => third-party}/rapidjson/pointer.h | 0 {src => third-party}/rapidjson/prettywriter.h | 0 {src => third-party}/rapidjson/rapidjson.h | 0 {src => third-party}/rapidjson/reader.h | 0 {src => third-party}/rapidjson/schema.h | 0 {src => third-party}/rapidjson/stream.h | 0 {src => third-party}/rapidjson/stringbuffer.h | 0 {src => third-party}/rapidjson/writer.h | 0 tools/codegen/amalgamate.py | 83 --- tools/codegen/gentable.py | 540 ------------------ tools/codegen/templates/amalgamation.cpp.in | 27 - tools/codegen/templates/blacklist.cpp.in | 13 - tools/codegen/templates/default.cpp.in | 189 ------ tools/codegen/templates/foreign.cpp.in | 73 --- tools/codegen/templates/typed_row.h.in | 103 ---- 44 files changed, 3 insertions(+), 1046 deletions(-) rename {policy => doc/policy}/bluetooth.md (100%) rename {src => third-party}/rapidjson/allocators.h (100%) rename {src => third-party}/rapidjson/document.h (100%) rename {src => third-party}/rapidjson/encodedstream.h (100%) rename {src => third-party}/rapidjson/encodings.h (100%) rename {src => third-party}/rapidjson/error/en.h (100%) rename {src => third-party}/rapidjson/error/error.h (100%) rename {src => third-party}/rapidjson/filereadstream.h (100%) rename {src => third-party}/rapidjson/filewritestream.h (100%) rename {src => third-party}/rapidjson/fwd.h (100%) rename {src => third-party}/rapidjson/internal/biginteger.h (100%) rename {src => third-party}/rapidjson/internal/diyfp.h (100%) rename {src => third-party}/rapidjson/internal/dtoa.h (100%) rename {src => third-party}/rapidjson/internal/ieee754.h (100%) rename {src => third-party}/rapidjson/internal/itoa.h (100%) rename {src => third-party}/rapidjson/internal/meta.h (100%) rename {src => third-party}/rapidjson/internal/pow10.h (100%) rename {src => third-party}/rapidjson/internal/regex.h (100%) rename {src => third-party}/rapidjson/internal/stack.h (100%) rename {src => third-party}/rapidjson/internal/strfunc.h (100%) rename {src => third-party}/rapidjson/internal/strtod.h (100%) rename {src => third-party}/rapidjson/internal/swap.h (100%) rename {src => third-party}/rapidjson/istreamwrapper.h (100%) rename {src => third-party}/rapidjson/memorybuffer.h (100%) rename {src => third-party}/rapidjson/memorystream.h (100%) rename {src => third-party}/rapidjson/msinttypes/inttypes.h (100%) rename {src => third-party}/rapidjson/msinttypes/stdint.h (100%) rename {src => third-party}/rapidjson/ostreamwrapper.h (100%) rename {src => third-party}/rapidjson/pointer.h (100%) rename {src => third-party}/rapidjson/prettywriter.h (100%) rename {src => third-party}/rapidjson/rapidjson.h (100%) rename {src => third-party}/rapidjson/reader.h (100%) rename {src => third-party}/rapidjson/schema.h (100%) rename {src => third-party}/rapidjson/stream.h (100%) rename {src => third-party}/rapidjson/stringbuffer.h (100%) rename {src => third-party}/rapidjson/writer.h (100%) delete mode 100755 tools/codegen/amalgamate.py delete mode 100755 tools/codegen/gentable.py delete mode 100644 tools/codegen/templates/amalgamation.cpp.in delete mode 100644 tools/codegen/templates/blacklist.cpp.in delete mode 100644 tools/codegen/templates/default.cpp.in delete mode 100644 tools/codegen/templates/foreign.cpp.in delete mode 100644 tools/codegen/templates/typed_row.h.in diff --git a/CMakeLists.txt b/CMakeLists.txt index d28a2bd..61de846 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved +# Copyright (c) 2019-present Samsung Electronics Co., Ltd All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,30 +35,15 @@ SET(CMAKE_SHARED_LINKER_FLAGS "-Wl,--as-needed") SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed") ADD_DEFINITIONS("-fPIC") -ADD_DEFINITIONS("-Wno-ignored-qualifiers") # Suppress osquery error - -# Set various platform/platform-version/build version/etc defines. -ADD_DEFINITIONS(-DLINUX="1" - -DPOSIX="1" - -DOSQUERY_LINUX="1" - -DOSQUERY_POSIX="1" - -DOSQUERY_BUILD_PLATFORM="linux" - -DOSQUERY_BUILD_DISTRO="linux") - -ADD_DEFINITIONS(-DOSQUERY_VERSION=${OSQUERY_VERSION} - -DOSQUERY_BUILD_VERSION=${OSQUERY_VERSION} - -DOSQUERY_BUILD_SDK_VERSION=${OSQUERY_VERSION}) +INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/src") # Suppresse SYSTEM header's warnings: -INCLUDE_DIRECTORIES(SYSTEM "${CMAKE_SOURCE_DIR}/src") +INCLUDE_DIRECTORIES(SYSTEM "${CMAKE_SOURCE_DIR}/third-party") INCLUDE_DIRECTORIES(SYSTEM "${CMAKE_SOURCE_DIR}/src/osquery/include") INCLUDE_DIRECTORIES("/usr/local/include") ENABLE_TESTING() -# Make sure the generated paths exist -EXECUTE_PROCESS(COMMAND mkdir -p "${CMAKE_BINARY_DIR}/generated") - ADD_SUBDIRECTORY(data) ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(plugins) diff --git a/policy/bluetooth.md b/doc/policy/bluetooth.md similarity index 100% rename from policy/bluetooth.md rename to doc/policy/bluetooth.md diff --git a/src/rapidjson/allocators.h b/third-party/rapidjson/allocators.h similarity index 100% rename from src/rapidjson/allocators.h rename to third-party/rapidjson/allocators.h diff --git a/src/rapidjson/document.h b/third-party/rapidjson/document.h similarity index 100% rename from src/rapidjson/document.h rename to third-party/rapidjson/document.h diff --git a/src/rapidjson/encodedstream.h b/third-party/rapidjson/encodedstream.h similarity index 100% rename from src/rapidjson/encodedstream.h rename to third-party/rapidjson/encodedstream.h diff --git a/src/rapidjson/encodings.h b/third-party/rapidjson/encodings.h similarity index 100% rename from src/rapidjson/encodings.h rename to third-party/rapidjson/encodings.h diff --git a/src/rapidjson/error/en.h b/third-party/rapidjson/error/en.h similarity index 100% rename from src/rapidjson/error/en.h rename to third-party/rapidjson/error/en.h diff --git a/src/rapidjson/error/error.h b/third-party/rapidjson/error/error.h similarity index 100% rename from src/rapidjson/error/error.h rename to third-party/rapidjson/error/error.h diff --git a/src/rapidjson/filereadstream.h b/third-party/rapidjson/filereadstream.h similarity index 100% rename from src/rapidjson/filereadstream.h rename to third-party/rapidjson/filereadstream.h diff --git a/src/rapidjson/filewritestream.h b/third-party/rapidjson/filewritestream.h similarity index 100% rename from src/rapidjson/filewritestream.h rename to third-party/rapidjson/filewritestream.h diff --git a/src/rapidjson/fwd.h b/third-party/rapidjson/fwd.h similarity index 100% rename from src/rapidjson/fwd.h rename to third-party/rapidjson/fwd.h diff --git a/src/rapidjson/internal/biginteger.h b/third-party/rapidjson/internal/biginteger.h similarity index 100% rename from src/rapidjson/internal/biginteger.h rename to third-party/rapidjson/internal/biginteger.h diff --git a/src/rapidjson/internal/diyfp.h b/third-party/rapidjson/internal/diyfp.h similarity index 100% rename from src/rapidjson/internal/diyfp.h rename to third-party/rapidjson/internal/diyfp.h diff --git a/src/rapidjson/internal/dtoa.h b/third-party/rapidjson/internal/dtoa.h similarity index 100% rename from src/rapidjson/internal/dtoa.h rename to third-party/rapidjson/internal/dtoa.h diff --git a/src/rapidjson/internal/ieee754.h b/third-party/rapidjson/internal/ieee754.h similarity index 100% rename from src/rapidjson/internal/ieee754.h rename to third-party/rapidjson/internal/ieee754.h diff --git a/src/rapidjson/internal/itoa.h b/third-party/rapidjson/internal/itoa.h similarity index 100% rename from src/rapidjson/internal/itoa.h rename to third-party/rapidjson/internal/itoa.h diff --git a/src/rapidjson/internal/meta.h b/third-party/rapidjson/internal/meta.h similarity index 100% rename from src/rapidjson/internal/meta.h rename to third-party/rapidjson/internal/meta.h diff --git a/src/rapidjson/internal/pow10.h b/third-party/rapidjson/internal/pow10.h similarity index 100% rename from src/rapidjson/internal/pow10.h rename to third-party/rapidjson/internal/pow10.h diff --git a/src/rapidjson/internal/regex.h b/third-party/rapidjson/internal/regex.h similarity index 100% rename from src/rapidjson/internal/regex.h rename to third-party/rapidjson/internal/regex.h diff --git a/src/rapidjson/internal/stack.h b/third-party/rapidjson/internal/stack.h similarity index 100% rename from src/rapidjson/internal/stack.h rename to third-party/rapidjson/internal/stack.h diff --git a/src/rapidjson/internal/strfunc.h b/third-party/rapidjson/internal/strfunc.h similarity index 100% rename from src/rapidjson/internal/strfunc.h rename to third-party/rapidjson/internal/strfunc.h diff --git a/src/rapidjson/internal/strtod.h b/third-party/rapidjson/internal/strtod.h similarity index 100% rename from src/rapidjson/internal/strtod.h rename to third-party/rapidjson/internal/strtod.h diff --git a/src/rapidjson/internal/swap.h b/third-party/rapidjson/internal/swap.h similarity index 100% rename from src/rapidjson/internal/swap.h rename to third-party/rapidjson/internal/swap.h diff --git a/src/rapidjson/istreamwrapper.h b/third-party/rapidjson/istreamwrapper.h similarity index 100% rename from src/rapidjson/istreamwrapper.h rename to third-party/rapidjson/istreamwrapper.h diff --git a/src/rapidjson/memorybuffer.h b/third-party/rapidjson/memorybuffer.h similarity index 100% rename from src/rapidjson/memorybuffer.h rename to third-party/rapidjson/memorybuffer.h diff --git a/src/rapidjson/memorystream.h b/third-party/rapidjson/memorystream.h similarity index 100% rename from src/rapidjson/memorystream.h rename to third-party/rapidjson/memorystream.h diff --git a/src/rapidjson/msinttypes/inttypes.h b/third-party/rapidjson/msinttypes/inttypes.h similarity index 100% rename from src/rapidjson/msinttypes/inttypes.h rename to third-party/rapidjson/msinttypes/inttypes.h diff --git a/src/rapidjson/msinttypes/stdint.h b/third-party/rapidjson/msinttypes/stdint.h similarity index 100% rename from src/rapidjson/msinttypes/stdint.h rename to third-party/rapidjson/msinttypes/stdint.h diff --git a/src/rapidjson/ostreamwrapper.h b/third-party/rapidjson/ostreamwrapper.h similarity index 100% rename from src/rapidjson/ostreamwrapper.h rename to third-party/rapidjson/ostreamwrapper.h diff --git a/src/rapidjson/pointer.h b/third-party/rapidjson/pointer.h similarity index 100% rename from src/rapidjson/pointer.h rename to third-party/rapidjson/pointer.h diff --git a/src/rapidjson/prettywriter.h b/third-party/rapidjson/prettywriter.h similarity index 100% rename from src/rapidjson/prettywriter.h rename to third-party/rapidjson/prettywriter.h diff --git a/src/rapidjson/rapidjson.h b/third-party/rapidjson/rapidjson.h similarity index 100% rename from src/rapidjson/rapidjson.h rename to third-party/rapidjson/rapidjson.h diff --git a/src/rapidjson/reader.h b/third-party/rapidjson/reader.h similarity index 100% rename from src/rapidjson/reader.h rename to third-party/rapidjson/reader.h diff --git a/src/rapidjson/schema.h b/third-party/rapidjson/schema.h similarity index 100% rename from src/rapidjson/schema.h rename to third-party/rapidjson/schema.h diff --git a/src/rapidjson/stream.h b/third-party/rapidjson/stream.h similarity index 100% rename from src/rapidjson/stream.h rename to third-party/rapidjson/stream.h diff --git a/src/rapidjson/stringbuffer.h b/third-party/rapidjson/stringbuffer.h similarity index 100% rename from src/rapidjson/stringbuffer.h rename to third-party/rapidjson/stringbuffer.h diff --git a/src/rapidjson/writer.h b/third-party/rapidjson/writer.h similarity index 100% rename from src/rapidjson/writer.h rename to third-party/rapidjson/writer.h diff --git a/tools/codegen/amalgamate.py b/tools/codegen/amalgamate.py deleted file mode 100755 index a785770..0000000 --- a/tools/codegen/amalgamate.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2014-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed in accordance with the terms specified in -# the LICENSE file found in the root directory of this source tree. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import jinja2 -import os -import sys - - -TEMPLATE_NAME = "amalgamation.cpp.in" -BEGIN_LINE = "/// BEGIN[GENTABLE]" -END_LINE = "/// END[GENTABLE]" - - -def genTableData(filename): - with open(filename, "rU") as fh: - data = fh.read() - begin_table = False - table_data = [] - for line in data.split("\n"): - if line.find(BEGIN_LINE) >= 0: - begin_table = True - elif line.find(END_LINE) >= 0: - begin_table = False - elif begin_table: - table_data.append(line) - if len(table_data) == 0: - return None - return "\n".join(table_data) - - -def main(argc, argv): - parser = argparse.ArgumentParser( - "Generate C++ amalgamation from C++ Table Plugin targets") - parser.add_argument("--foreign", default=False, action="store_true", - help="Generate a foreign table set amalgamation") - parser.add_argument("--templates", - help="Path to codegen output .cpp.in templates") - parser.add_argument("--category", help="Category name of generated tables") - parser.add_argument("--sources", - help="Path to the folder containing the .cpp files") - parser.add_argument("--output", help="Path to the output .cpp files") - args = parser.parse_args() - - tables = [] - # Discover the output template, usually a black cpp file with includes. - template = os.path.join(args.templates, TEMPLATE_NAME) - with open(template, "rU") as fh: - template_data = fh.read() - - for base, _, filenames in os.walk(args.sources): - for filename in filenames: - if filename == args.category: - continue - table_data = genTableData(os.path.join(base, filename)) - if table_data is not None: - tables.append(table_data) - - env = jinja2.Environment(keep_trailing_newline=True) - amalgamation = env.from_string(template_data).render(tables=tables, - foreign=args.foreign) - try: - os.makedirs(os.path.dirname(args.output)) - except OSError: - # Generated folder already exists - pass - with open(args.output, "w") as fh: - fh.write(amalgamation) - return 0 - - -if __name__ == "__main__": - exit(main(len(sys.argv), sys.argv)) diff --git a/tools/codegen/gentable.py b/tools/codegen/gentable.py deleted file mode 100755 index b61a91f..0000000 --- a/tools/codegen/gentable.py +++ /dev/null @@ -1,540 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2014-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed in accordance with the terms specified in -# the LICENSE file found in the root directory of this source tree. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import ast -import fnmatch -import jinja2 -import logging -import os -import sys - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# the log format for the logging module -LOG_FORMAT = "%(levelname)s [Line %(lineno)d]: %(message)s" - -# Read all implementation templates -TEMPLATES = {} - -# Temporary reserved column names -RESERVED = ["n", "index"] - -# Set the platform in osquery-language -PLATFORM = "linux" - -# Supported SQL types for spec -class DataType(object): - def __init__(self, affinity, cpp_type="std::string"): - '''A column datatype is a pair of a SQL affinity to C++ type.''' - self.affinity = affinity - self.type = cpp_type - - def __repr__(self): - return self.affinity - -# Define column-type MACROs for the table specs -TEXT = DataType("TEXT_TYPE") -DATE = DataType("TEXT_TYPE") -DATETIME = DataType("TEXT_TYPE") -INTEGER = DataType("INTEGER_TYPE", "int") -BIGINT = DataType("BIGINT_TYPE", "long long int") -UNSIGNED_BIGINT = DataType("UNSIGNED_BIGINT_TYPE", "long long unsigned int") -DOUBLE = DataType("DOUBLE_TYPE", "double") -BLOB = DataType("BLOB_TYPE", "Blob") - -# Define table-category MACROS from the table specs -UNKNOWN = "UNKNOWN" -UTILITY = "UTILITY" -SYSTEM = "SYSTEM" -NETWORK = "NETWORK" -EVENTS = "EVENTS" -APPLICATION = "APPLICATION" - -# This should mimic the C++ enumeration ColumnOptions in table.h -COLUMN_OPTIONS = { - "index": "INDEX", - "additional": "ADDITIONAL", - "required": "REQUIRED", - "optimized": "OPTIMIZED", - "hidden": "HIDDEN", -} - -# Column options that render tables uncacheable. -NON_CACHEABLE = [ - "REQUIRED", - "ADDITIONAL", - "OPTIMIZED", -] - -TABLE_ATTRIBUTES = { - "event_subscriber": "EVENT_BASED", - "user_data": "USER_BASED", - "cacheable": "CACHEABLE", - "utility": "UTILITY", - "kernel_required": "KERNEL_REQUIRED", -} - - -def WINDOWS(): - return PLATFORM in ['windows', 'win32', 'cygwin'] - - -def LINUX(): - return PLATFORM in ['linux'] - - -def POSIX(): - return PLATFORM in ['linux', 'darwin', 'freebsd'] - - -def DARWIN(): - return PLATFORM in ['darwin'] - - -def FREEBSD(): - return PLATFORM in ['freebsd'] - - -def to_camel_case(snake_case): - """ convert a snake_case string to camelCase """ - components = snake_case.split('_') - return components[0] + "".join(x.title() for x in components[1:]) - -def to_upper_camel_case(snake_case): - """ convert a snake_case string to UpperCamelCase """ - components = snake_case.split('_') - return "".join(x.title() for x in components) - -def lightred(msg): - return "\033[1;31m %s \033[0m" % str(msg) - - -def is_blacklisted(table_name, path=None, blacklist=None): - """Allow blacklisting by tablename.""" - if blacklist is None: - specs_path = os.path.dirname(path) - if os.path.basename(specs_path) != "specs": - specs_path = os.path.dirname(specs_path) - blacklist_path = os.path.join(specs_path, "blacklist") - if not os.path.exists(blacklist_path): - return False - try: - with open(blacklist_path, "r") as fh: - blacklist = [ - line.strip() for line in fh.read().split("\n") - if len(line.strip()) > 0 and line.strip()[0] != "#" - ] - except: - # Blacklist is not readable. - return False - if not blacklist: - return False - - # table_name based blacklisting! - for item in blacklist: - item = item.split(":") - # If this item is restricted to a platform and the platform - # and table name match - if len(item) > 1 and PLATFORM == item[0] and table_name == item[1]: - return True - elif len(item) == 1 and table_name == item[0]: - return True - return False - - -def setup_templates(templates_path): - if not os.path.exists(templates_path): - templates_path = os.path.join( - os.path.dirname(tables_path), "templates") - if not os.path.exists(templates_path): - print("Cannot read templates path: %s" % (templates_path)) - exit(1) - templates = (f for f in os.listdir(templates_path) if fnmatch.fnmatch(f, "*.in")) - for template in templates: - template_name = template.split(".", 1)[0] - with open(os.path.join(templates_path, template), "r") as fh: - TEMPLATES[template_name] = fh.read().replace("\\\n", "") - - -class Singleton(object): - - """ - Make sure that anything that subclasses Singleton can only be instantiated - once - """ - - _instance = None - - def __new__(self, *args, **kwargs): - if not self._instance: - self._instance = super(Singleton, self).__new__( - self, *args, **kwargs) - return self._instance - - -class TableState(Singleton): - - """ - Maintain the state of of the table commands during the execution of - the config file - """ - - def __init__(self): - self.table_name = "" - self.schema = [] - self.header = "" - self.impl = "" - self.function = "" - self.function_delete = "" - self.function_insert = "" - self.function_update = "" - self.class_name = "" - self.description = "" - self.attributes = {} - self.examples = [] - self.aliases = [] - self.fuzz_paths = [] - self.has_options = False - self.has_column_aliases = False - self.strongly_typed_rows = False - self.generator = False - - def columns(self): - return [i for i in self.schema if isinstance(i, Column)] - - def foreign_keys(self): - return [i for i in self.schema if isinstance(i, ForeignKey)] - - def generate(self, path, template="default"): - """Generate the virtual table files""" - logging.debug("TableState.generate") - - all_options = [] - # Create a list of column options from the kwargs passed to the column. - for column in self.columns(): - column_options = [] - for option in column.options: - # Only allow explicitly-defined options. - if option in COLUMN_OPTIONS: - column_options.append("ColumnOptions::" + COLUMN_OPTIONS[option]) - all_options.append(COLUMN_OPTIONS[option]) - else: - print(yellow( - "Table %s column %s contains an unknown option: %s" % ( - self.table_name, column.name, option))) - column.options_set = " | ".join(column_options) - if len(column.aliases) > 0: - self.has_column_aliases = True - if len(all_options) > 0: - self.has_options = True - if "event_subscriber" in self.attributes: - self.generator = True - if "strongly_typed_rows" in self.attributes: - self.strongly_typed_rows = True - if "cacheable" in self.attributes: - if self.generator: - print(lightred( - "Table cannot use a generator and be marked cacheable: %s" % (path))) - exit(1) - if self.table_name == "" or self.function == "": - print(lightred("Invalid table spec: %s" % (path))) - exit(1) - - # Check for reserved column names - for column in self.columns(): - if column.name in RESERVED: - print(lightred(("Cannot use column name: %s in table: %s " - "(the column name is reserved)" % ( - column.name, self.table_name)))) - exit(1) - - if "ADDITIONAL" in all_options and "INDEX" not in all_options: - if "no_pkey" not in self.attributes: - print(lightred( - "Table cannot have 'additional' columns without an index: %s" %( - path))) - exit(1) - - path_bits = path.split("/") - for i in range(1, len(path_bits)): - dir_path = "" - for j in range(i): - dir_path += "%s/" % path_bits[j] - if not os.path.exists(dir_path): - try: - os.mkdir(dir_path) - except: - # May encounter a race when using a make jobserver. - pass - logging.debug("generating %s" % path) - self.impl_content = jinja2.Template(TEMPLATES[template]).render( - table_name=self.table_name, - table_name_cc=to_camel_case(self.table_name), - table_name_ucc=to_upper_camel_case(self.table_name), - schema=self.columns(), - header=self.header, - impl=self.impl, - function=self.function, - function_delete=self.function_delete, - function_insert=self.function_insert, - function_update=self.function_update, - class_name=self.class_name, - attributes=self.attributes, - examples=self.examples, - aliases=self.aliases, - has_options=self.has_options, - has_column_aliases=self.has_column_aliases, - generator=self.generator, - strongly_typed_rows=self.strongly_typed_rows, - attribute_set=[TABLE_ATTRIBUTES[attr] for attr in self.attributes if attr in TABLE_ATTRIBUTES], - ) - - with open(path, "w+") as file_h: - file_h.write(self.impl_content) - - def blacklist(self, path): - print(lightred("Blacklisting generated %s" % path)) - logging.debug("blacklisting %s" % path) - self.generate(path, template="blacklist") - -table = TableState() - - -class Column(object): - - """ - Part of an osquery table schema. - Define a column by name and type with an optional description to assist - documentation generation and reference. - """ - - def __init__(self, name, col_type, description="", aliases=[], **kwargs): - self.name = name - self.type = col_type - self.description = description - self.aliases = aliases - self.options = kwargs - - -class ForeignKey(object): - - """ - Part of an osquery table schema. - Loosely define a column in a table spec as a Foreign key in another table. - """ - - def __init__(self, **kwargs): - self.column = kwargs.get("column", "") - self.table = kwargs.get("table", "") - - -def table_name(name, aliases=[]): - """define the virtual table name""" - logging.debug("- table_name") - logging.debug(" - called with: %s" % name) - table.table_name = name - table.description = "" - table.attributes = {} - table.examples = [] - table.aliases = aliases - - -def schema(schema_list): - """ - define a list of Column object which represent the columns of your virtual - table - """ - logging.debug("- schema") - for it in schema_list: - if isinstance(it, Column): - logging.debug(" - column: %s (%s)" % (it.name, it.type)) - if isinstance(it, ForeignKey): - logging.debug(" - foreign_key: %s (%s)" % (it.column, it.table)) - table.schema = schema_list - - -def extended_schema(check, schema_list): - """ - define a comparator and a list of Columns objects. - """ - logging.debug("- extended schema") - for it in schema_list: - if isinstance(it, Column): - logging.debug(" - column: %s (%s)" % (it.name, it.type)) - if not check(): - it.options['hidden'] = True - table.schema.append(it) - - -def description(text): - if text[-1:] != '.': - print(lightred("Table description must end with a period!")) - exit(1) - table.description = text - - -def select_all(name=None): - if name is None: - name = table.table_name - return "select count(*) from %s;" % (name) - - -def examples(example_queries): - table.examples = example_queries - - -def attributes(**kwargs): - for attr in kwargs: - table.attributes[attr] = kwargs[attr] - - -def fuzz_paths(paths): - table.fuzz_paths = paths - - -def implementation(impl_string, generator=False): - """ - define the path to the implementation file and the function which - implements the virtual table. You should use the following format: - - # the path is "osquery/table/implementations/foo.cpp" - # the function is "QueryData genFoo();" - implementation("foo@genFoo") - """ - logging.debug("- implementation") - filename, function = impl_string.split("@") - class_parts = function.split("::")[::-1] - function = class_parts[0] - class_name = class_parts[1] if len(class_parts) > 1 else "" - impl = "%s.cpp" % filename - logging.debug(" - impl => %s" % impl) - logging.debug(" - function => %s" % function) - logging.debug(" - class_name => %s" % class_name) - table.impl = impl - table.function = function - table.class_name = class_name - table.generator = generator - - '''Check if the table has a subscriber attribute, if so, enforce time.''' - if "event_subscriber" in table.attributes: - if not table.table_name.endswith("_events"): - print(lightred("Event subscriber must use a '_events' suffix")) - sys.exit(1) - columns = {} - # There is no dictionary comprehension on all supported platforms. - for column in table.schema: - if isinstance(column, Column): - columns[column.name] = column.type - if "time" not in columns: - print(lightred("Event subscriber: %s needs a 'time' column." % ( - table.table_name))) - sys.exit(1) - if columns["time"] is not BIGINT: - print(lightred( - "Event subscriber: %s, 'time' column must be a %s type" % ( - table.table_name, BIGINT))) - sys.exit(1) - -### patch start ########################################################### -def implementation_delete(impl_string, generator=False): - if impl_string is None: - table.function_delete = "" - else: - filename, function_delete = impl_string.split("@") - class_parts = function_delete.split("::")[::-1] - function_delete = class_parts[0] - class_name = class_parts[1] if len(class_parts) > 1 else "" - impl = "%s.cpp" % filename - table.impl = impl - table.function_delete = function_delete - table.class_name = class_name - table.generator = generator - -def implementation_insert(impl_string, generator=False): - if impl_string is None: - table.function_insert = "" - else: - filename, function_insert = impl_string.split("@") - class_parts = function_insert.split("::")[::-1] - function_insert = class_parts[0] - class_name = class_parts[1] if len(class_parts) > 1 else "" - impl = "%s.cpp" % filename - table.impl = impl - table.function_insert = function_insert - table.class_name = class_name - table.generator = generator - -def implementation_update(impl_string, generator=False): - if impl_string is None: - table.function_update = "" - else: - filename, function_update = impl_string.split("@") - class_parts = function_update.split("::")[::-1] - function_update = class_parts[0] - class_name = class_parts[1] if len(class_parts) > 1 else "" - impl = "%s.cpp" % filename - table.impl = impl - table.function_update = function_update - table.class_name = class_name - table.generator = generator - -### patch end ############################################################# - -def main(): - parser = argparse.ArgumentParser( - "Generate C++ Table Plugin from specfile.") - parser.add_argument( - "--debug", default=False, action="store_true", - help="Output debug messages (when developing)" - ) - parser.add_argument("--disable-blacklist", default=False, - action="store_true") - parser.add_argument("--header", default=False, action="store_true", - help="Generate the header file instead of cpp") - parser.add_argument("--foreign", default=False, action="store_true", - help="Generate a foreign table") - parser.add_argument("--templates", default=SCRIPT_DIR + "/templates", - help="Path to codegen output .cpp.in templates") - parser.add_argument("spec_file", help="Path to input .table spec file") - parser.add_argument("output", help="Path to output .cpp file") - args = parser.parse_args() - - if args.debug: - logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) - else: - logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) - - filename = args.spec_file - output = args.output - if filename.endswith(".table"): - # Adding a 3rd parameter will enable the blacklist - - setup_templates(args.templates) - with open(filename, "r") as file_handle: - tree = ast.parse(file_handle.read()) - exec(compile(tree, "", "exec")) - blacklisted = is_blacklisted(table.table_name, path=filename) - if not args.disable_blacklist and blacklisted: - table.blacklist(output) - else: - if args.header: - template_type = "typed_row" - elif args.foreign: - template_type = "foreign" - else: - template_type = "default" - table.generate(output, template=template_type) - -if __name__ == "__main__": - main() diff --git a/tools/codegen/templates/amalgamation.cpp.in b/tools/codegen/templates/amalgamation.cpp.in deleted file mode 100644 index 519886c..0000000 --- a/tools/codegen/templates/amalgamation.cpp.in +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -/* -** This file is generated. Do not modify it manually! -*/ - -#include -#include -#include - -namespace osquery { -{% if foreign %} -void registerForeignTables() { -{% endif %} -{% for table in tables %} -{{table}} -{% endfor %} -{% if foreign %} -} -{% endif %} -} diff --git a/tools/codegen/templates/blacklist.cpp.in b/tools/codegen/templates/blacklist.cpp.in deleted file mode 100644 index 3a27c60..0000000 --- a/tools/codegen/templates/blacklist.cpp.in +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -/* -** This file is generated. Do not modify it manually! -*/ - -void __blacklisted_{{table_name}}() {} diff --git a/tools/codegen/templates/default.cpp.in b/tools/codegen/templates/default.cpp.in deleted file mode 100644 index e463269..0000000 --- a/tools/codegen/templates/default.cpp.in +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ - -/* -** This file is generated. Do not modify it manually! -*/ - -#include -#include - -namespace osquery { - -/// BEGIN[GENTABLE] -namespace tables { -{% if class_name == "" %}\ -{% if generator %}\ -void {{function}}(RowYield& yield, QueryContext& context); -{% elif strongly_typed_rows %}\ -osquery::TableRows {{function}}(QueryContext& context); -{% else %}\ -osquery::QueryData {{function}}(QueryContext& context); - -/// patch start ////////////////////////////////////////////////// -{% if function_delete != "" %}\ -osquery::QueryData {{function_delete}}(QueryContext& context, - const PluginRequest& request); -{% endif %}\ - -{% if function_insert != "" %}\ -osquery::QueryData {{function_insert}}(QueryContext& context, - const PluginRequest& request); -{% endif %}\ - -{% if function_update != "" %}\ -osquery::QueryData {{function_update}}(QueryContext& context, - const PluginRequest& request); -{% endif %}\ - -/// patch end /////////////////////////////////////////////////// - -{% endif %}\ -{% else %} -class {{class_name}} { - public: - void {{function}}(RowYield& yield, QueryContext& context); -}; -{% endif %}\ -} - -class {{table_name_cc}}TablePlugin : public TablePlugin { - private: - TableColumns columns() const override { - return { -{% for column in schema %}\ - std::make_tuple("{{column.name}}", {{column.type.affinity}},\ -{% if column.options|length > 0 %} {{column.options_set}}\ -{% else %} ColumnOptions::DEFAULT\ -{% endif %}\ -), -{% endfor %}\ - }; - } -{% if aliases|length > 0 %}\ - - std::vector aliases() const override { - return { -{% for alias in aliases %}\ - "{{alias}}", -{% endfor %}\ - }; - } -{% endif %}\ - -{% if has_column_aliases %}\ - - ColumnAliasSet columnAliases() const override { - return { -{% for column in schema %}\ -{% if column.aliases|length > 0 %}\ - {"{{column.name}}", {% raw %}{{% endraw %}\ -{% for alias in column.aliases %}"{{alias}}"\ -{% if not loop.last %}, {% endif %}\ -{% endfor %}}}, -{% endif %}\ -{% endfor %}\ - }; - } - - AliasColumnMap aliasedColumns() const override { - return { -{% for column in schema %}\ -{% if column.aliases|length > 0 %}\ -{% for alias in column.aliases %}\ - { "{{alias}}", "{{column.name}}" }, -{% endfor %}\ -{% endif %}\ -{% endfor %}\ - }; - } -{% endif %}\ - - TableAttributes attributes() const override { - return \ -{% for attribute in attribute_set %}\ - TableAttributes::{{attribute}} |\ -{% endfor %}\ - TableAttributes::NONE; - } - -{% if generator %}\ - bool usesGenerator() const override { return true; } - - void generator(RowYield& yield, QueryContext& context) override { - tables::{{function}}(yield, context); - } -{% else %}\ - TableRows generate(QueryContext& context) override { -{% if attributes.cacheable %}\ - if (isCached(kCacheStep, context)) { - return getCache(); - } -{% endif %}\ -{% if attributes.strongly_typed_rows %}\ - TableRows results = tables::{{function}}(context); -{% else %}\ - TableRows results = osquery::tableRowsFromQueryData(tables::{{function}}(context)); -{% endif %} -{% if attributes.cacheable %}\ - setCache(kCacheStep, kCacheInterval, context, results); -{% endif %} - return results; - } - -/// patch start ///////////////////////////////////////////////////// -{% if function_delete != "" %}\ - QueryData delete_(QueryContext& context, - const PluginRequest& request) override { - return tables::{{function_delete}}(context, request); - } -{% endif %}\ - -{% if function_insert != "" %}\ - QueryData insert(QueryContext& context, - const PluginRequest& request) override { - return tables::{{function_insert}}(context, request); - } -{% endif %}\ - -{% if function_update != "" %}\ - QueryData update(QueryContext& context, - const PluginRequest& request) override { - return tables::{{function_update}}(context, request); - } -{% endif %}\ - -/// patch end /////////////////////////////////////////////////////// - -{% endif %}\ - -}; - -{% if attributes.utility %} -REGISTER_INTERNAL({{table_name_cc}}TablePlugin, "table", "{{table_name}}"); -{% else %} -REGISTER({{table_name_cc}}TablePlugin, "table", "{{table_name}}"); -{% endif %} -/// END[GENTABLE] - -} diff --git a/tools/codegen/templates/foreign.cpp.in b/tools/codegen/templates/foreign.cpp.in deleted file mode 100644 index e62c73d..0000000 --- a/tools/codegen/templates/foreign.cpp.in +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -/* -** This file is generated. Do not modify it manually! -*/ - -#include -#include - -namespace osquery { -namespace tables { - -auto {{table_name_cc}}Register = []() { -/// BEGIN[GENTABLE] - class {{table_name_cc}}TablePlugin : public TablePlugin { - private: - TableColumns columns() const override { - return { -{% for column in schema %}\ - std::make_tuple("{{column.name}}", {{column.type.affinity}},\ -{% if column.options|length > 0 %} {{column.options_set}}\ -{% else %} ColumnOptions::DEFAULT\ -{% endif %}\ -), -{% endfor %}\ - }; - } -{% if aliases|length > 0 %}\ - - std::vector aliases() const override { - return { -{% for alias in aliases %}\ - "{{alias}}", -{% endfor %}\ - }; - } -{% endif %}\ - -{% if has_column_aliases %}\ - - ColumnAliasSet columnAliases() const override { - return { -{% for column in schema %}\ -{% if column.aliases|length > 0 %}\ - {"{{column.name}}", {% raw %}{{% endraw %}\ -{% for alias in column.aliases %}"{{alias}}"\ -{% if not loop.last %}, {% endif %}\ -{% endfor %}}}, -{% endif %}\ -{% endfor %}\ - }; - } -{% endif %}\ - - TableRows generate(QueryContext& request) override { return TableRows(); } - }; - - { - auto registry = RegistryFactory::get().registry("table"); - registry->add("{{table_name}}", - std::make_shared<{{table_name_cc}}TablePlugin>(), false); - } -/// END[GENTABLE] -}; - -} -} diff --git a/tools/codegen/templates/typed_row.h.in b/tools/codegen/templates/typed_row.h.in deleted file mode 100644 index 991b0b3..0000000 --- a/tools/codegen/templates/typed_row.h.in +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -/* -** This file is generated. Do not modify it manually! -*/ - -#include - -namespace osquery { -namespace tables { - -class {{table_name_ucc}}Row : public TableRow { -public: - {{table_name_ucc}}Row() { - } - -{% for column in schema %}\ - {{column.type.type}} {{column.name}}_col; -{% endfor %}\ - - enum Column { -{% for column in schema %}\ -{% if loop.index0 < 63 %}\ - {{column.name.upper()}} = 1ULL << {{loop.index0}}, -{% else %}\ - {{column.name.upper()}} = 1ULL << 63, -{% endif %}\ -{% endfor %}\ - }; - - virtual int get_rowid(sqlite_int64 default_value, sqlite_int64* pRowid) const override { -{% set filtered = schema|select("equalto", "rowid") %}\ -{% if filtered|list|length == 1 %}\ - *pRowid = rowid_col; -{% else %}\ - *pRowid = default_value; -{% endif %}\ - return SQLITE_OK; - } - - virtual int get_column(sqlite3_context* ctx, sqlite3_vtab* vtab, int col) override { - switch (col) { -{% for column in schema %}\ - case {{loop.index0}}: -{% if column.type.affinity == "TEXT_TYPE" %}\ - sqlite3_result_text(ctx, {{column.name}}_col.c_str(), static_cast({{column.name}}_col.size()), SQLITE_STATIC); -{% elif column.type.affinity == "INTEGER_TYPE" %}\ - sqlite3_result_int(ctx, {{column.name}}_col); -{% elif column.type.affinity == "BIGINT_TYPE" or column.type.affinity == "UNSIGNED_BIGINT_TYPE" %}\ - sqlite3_result_int64(ctx, {{column.name}}_col); -{% elif column.type.affinity == "DOUBLE_TYPE" %}\ - sqlite3_result_double(ctx, {{column.name}}_col); -{% endif %}\ - break; -{% endfor %}\ - } - return SQLITE_OK; - } - - virtual Status serialize(JSON& doc, rapidjson::Value& obj) const override { -{% for column in schema %}\ -{% if column.type.affinity == "TEXT_TYPE" %}\ - doc.addRef("{{column.name}}", {{column.name}}_col); -{% else %}\ - doc.add("{{column.name}}", {{column.name}}_col); -{% endif %}\ -{% endfor %}\ - - return Status(); - } - - virtual operator Row() const override { - Row result; - -{% for column in schema %}\ -{% if column.type.affinity == "TEXT_TYPE" %}\ - result["{{column.name}}"] = {{column.name}}_col; -{% elif column.type.affinity == "INTEGER_TYPE" %}\ - result["{{column.name}}"] = INTEGER({{column.name}}_col); -{% elif column.type.affinity == "BIGINT_TYPE" %}\ - result["{{column.name}}"] = BIGINT({{column.name}}_col); -{% elif column.type.affinity == "UNSIGNED_BIGINT_TYPE" %}\ - result["{{column.name}}"] = UNSIGNED_BIGINT({{column.name}}_col); -{% elif column.type.affinity == "DOUBLE_TYPE" %}\ - result["{{column.name}}"] = DOUBLE({{column.name}}_col); -{% endif %}\ -{% endfor %}\ - - return result; - } - - virtual TableRowHolder clone() const override { - return TableRowHolder(new {{table_name_ucc}}Row(*this)); - } -}; -} -} -- 2.34.1