diff --git a/Main.cpp b/Main.cpp index 782861f3..b9d50922 100644 --- a/Main.cpp +++ b/Main.cpp @@ -26,16 +26,23 @@ #include "oomd/Log.h" #include "oomd/Oomd.h" #include "oomd/PluginRegistry.h" +#include "oomd/config/CompactConfigParser.h" #include "oomd/config/ConfigCompiler.h" #include "oomd/config/JsonConfigParser.h" #include "oomd/include/Assert.h" #include "oomd/include/CgroupPath.h" #include "oomd/include/Defines.h" #include "oomd/util/Fs.h" +#include "oomd/util/Util.h" static constexpr auto kConfigFilePath = "/etc/oomd.json"; static constexpr auto kCgroupFsRoot = "/sys/fs/cgroup"; +enum class ConfigFormat { + JSON = 0, + COMPACT, +}; + static void printUsage() { std::cerr << "usage: oomd [OPTION]...\n\n" @@ -47,6 +54,10 @@ static void printUsage() { " --check-config, -c CONFIG Check config file (default: /etc/oomd.json)\n" " --list-plugins, -l List all available plugins\n" " --drop-in-dir, -w DIR Directory to watch for drop in configs\n" + " -Xoption=value Additional options\n" + "\n" + "additional options:\n" + " -Xconfig=[json|compact]\n" << std::endl; } @@ -74,17 +85,29 @@ static bool cgroup_fs_valid(std::string& path) { } static std::unique_ptr parseConfig( + ConfigFormat format, const std::string& flag_conf_file) { + std::unique_ptr ir; + std::ifstream conf_file(flag_conf_file, std::ios::in); if (!conf_file.is_open()) { - std::cerr << "Could not open confg_file=" << flag_conf_file << std::endl; + std::cerr << "Could not open config_file=" << flag_conf_file << std::endl; return nullptr; } std::stringstream buf; buf << conf_file.rdbuf(); - Oomd::Config2::JsonConfigParser json_parser; - auto ir = json_parser.parse(buf.str()); + + if (format == ConfigFormat::JSON) { + Oomd::Config2::JsonConfigParser json_parser; + ir = json_parser.parse(buf.str()); + } else if (format == ConfigFormat::COMPACT) { + Oomd::Config2::CompactConfigParser compact_parser; + ir = compact_parser.parse(buf.str()); + } else { + std::cerr << "Unhandled config parser format" << std::endl; + } + if (!ir) { std::cerr << "Could not parse conf_file=" << flag_conf_file << std::endl; return nullptr; @@ -93,17 +116,45 @@ static std::unique_ptr parseConfig( return ir; } +static int processAdditionalOptions( + const std::string& opt_pair, + ConfigFormat& config_format) { + auto parts = Oomd::Util::split(opt_pair, '='); + if (parts.size() != 2) { + std::cerr << "Invalid option=\"" << opt_pair << "\"" << std::endl; + return 1; + } + + for (auto& p : parts) { + Oomd::Util::strip(p); + } + + if (parts[0] == "config") { + if (parts[1] == "json") { + config_format = ConfigFormat::JSON; + } else if (parts[1] == "compact") { + config_format = ConfigFormat::COMPACT; + } else { + std::cerr << "Unrecognized config format=" << parts[1] << std::endl; + return 1; + } + } + + return 0; +} + int main(int argc, char** argv) { std::string flag_conf_file = kConfigFilePath; std::string cgroup_fs = kCgroupFsRoot; std::string drop_in_dir; int interval = 5; bool should_check_config = false; + ConfigFormat config_format = ConfigFormat::JSON; int option_index = 0; int c = 0; - const char* const short_options = "hC:w:i:f:c:l"; + const char* const short_options = "hC:w:i:f:c:lX:"; option long_options[] = { option{"help", no_argument, nullptr, 'h'}, option{"config", required_argument, nullptr, 'C'}, @@ -112,6 +163,7 @@ int main(int argc, char** argv) { option{"check-config", required_argument, nullptr, 'c'}, option{"list-plugins", no_argument, nullptr, 'l'}, option{"drop-in-dir", required_argument, nullptr, 'w'}, + option{"", required_argument, nullptr, 'X'}, option{nullptr, 0, nullptr, 0}}; while ((c = getopt_long( @@ -143,6 +195,11 @@ int main(int argc, char** argv) { case 'f': cgroup_fs = std::string(optarg); break; + case 'X': + if (processAdditionalOptions(std::string(optarg), config_format)) { + return 1; + } + break; case 0: break; case '?': @@ -176,7 +233,7 @@ int main(int argc, char** argv) { } if (should_check_config) { - auto ir = parseConfig(flag_conf_file); + auto ir = parseConfig(config_format, flag_conf_file); if (!ir) { return 1; } @@ -203,7 +260,7 @@ int main(int argc, char** argv) { std::cerr << "oomd running with conf_file=" << flag_conf_file << " interval=" << interval << std::endl; - auto ir = parseConfig(flag_conf_file); + auto ir = parseConfig(config_format, flag_conf_file); if (!ir) { return EXIT_CANT_RECOVER; } diff --git a/config/CompactConfigParser.cpp b/config/CompactConfigParser.cpp new file mode 100644 index 00000000..1c6e1c89 --- /dev/null +++ b/config/CompactConfigParser.cpp @@ -0,0 +1,341 @@ +/* + * Copyright (C) 2018-present, Facebook, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "oomd/config/CompactConfigParser.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "oomd/Log.h" +#include "oomd/include/Assert.h" +#include "oomd/util/Fs.h" +#include "oomd/util/Util.h" + +namespace { + +enum class ParseState { + INIT = 0, + SECTION_HEADER, + SECTION, + SECTION_ARGUMENTS_HEADER, + SECTION_ARGUMENTS, +}; + +struct ParsedSection { + bool isInit{false}; // true if struct has been populated + std::string name; + std::string plugin; + bool isDetector{true}; // false implies action + std::unordered_map args; + std::vector conditions; // only valid for actions + std::string chainedafter; // only valid for actions +}; + +bool isArgumentHeader(const std::string& line) { + auto l = line; // copy + Oomd::Util::strip(l); + return l == "[[args]]"; +} + +bool isSectionHeader(const std::string& line) { + auto l = line; // copy + Oomd::Util::strip(l); + if (l.size() < 2) { + return false; + } + + if (l[0] == '[' && l[l.size() - 1] == ']' && !isArgumentHeader(line)) { + return true; + } + + return false; +} + +/* + * Precondition is that @param line is a well formed section header + */ +std::string parseSectionName(const std::string& line) { + OCHECK_EXCEPT(line.size() >= 2, std::runtime_error("Invalid section header")); + auto l = line; + l.erase(0, 1); + l.pop_back(); + return l; +} + +std::optional> parseDirective( + const std::string& line) { + auto parts = Oomd::Util::split(line, '='); + if (parts.size() != 2) { + OLOG << "Invalid section directive=\"" << line << "\""; + return std::nullopt; + } + + for (auto& part : parts) { + Oomd::Util::strip(part); + } + + return std::make_pair( + std::move(parts[0]), std::move(parts[1])); +} + +bool isSectionValid(const ParsedSection& section) { + // Required fields + if (!section.isInit) { + OLOG << "Section=" << section.name << " is not initialized by parser"; + return false; + } + + if (section.name.empty()) { + OLOG << "Section mission name"; + return false; + } + + if (section.plugin.empty()) { + OLOG << "Section=" << section.name << " missing plugin"; + return false; + } + + // Check that conditions are only used with actions + if (section.isDetector && section.conditions.size()) { + OLOG << "Cannot use conditions with detector in section=" << section.name; + return false; + } + + // Check that chainedafter diretives are only used with actions + if (section.isDetector && section.chainedafter.size()) { + OLOG << "Cannot use chainedafter with detector in section=" << section.name; + return false; + } + + // Check that either chainedafter or conditions directive is used + if (!section.isDetector && + (section.chainedafter.empty() == section.conditions.empty())) { + OLOG << "Must use directive (chainedafter XOR condition) in section=" + << section.name; + return false; + } + + return true; +} + +std::optional> parseConfig( + const std::string& input) { + std::vector ret; + ParseState state = ParseState::INIT; + auto lines = Oomd::Util::split(input, '\n'); + ParsedSection section; + + for (const auto& line : lines) { + // Don't process blank lines + if (Oomd::Util::isBlank(line)) { + continue; + } + + // Perform state transitions as necessary + if (isSectionHeader(line)) { + // Only valid to transition from INIT or SECTION_ARGUMENTS + if (state != ParseState::INIT && state != ParseState::SECTION_ARGUMENTS) { + OLOG << "Unexpected section header=\"" << line << "\". " + << "Did you remember the section arguments?"; + return std::nullopt; + } + state = ParseState::SECTION_HEADER; + } else if (isArgumentHeader(line)) { + // Only valid to transition from SECTION state + if (state != ParseState::SECTION) { + OLOG << "Missing section header for argument list=\"" << line << "\""; + return std::nullopt; + } + state = ParseState::SECTION_ARGUMENTS_HEADER; + } else if (state == ParseState::SECTION_HEADER) { + state = ParseState::SECTION; + } else if (state == ParseState::SECTION_ARGUMENTS_HEADER) { + state = ParseState::SECTION_ARGUMENTS; + } + + // Handle each state. + // + // Don't do any state transitions down here. Do them above with the + // other state transitions. + switch (state) { + case ParseState::INIT: + // Invalid state + OLOG << "Could not find initial section header"; + return std::nullopt; + break; + case ParseState::SECTION_HEADER: + // Store previous section + if (section.isInit) { + ret.emplace_back(std::move(section)); + } + section = {}; + section.isInit = true; + section.name = parseSectionName(line); + break; + case ParseState::SECTION: { + auto pair = parseDirective(line); + if (!pair.has_value()) { + OLOG << "Invalid directive format in section=" << section.name; + return std::nullopt; + } + + if (pair->first == "detector") { + section.isDetector = true; + section.plugin = std::move(pair->second); + } else if (pair->first == "action") { + section.isDetector = false; + section.plugin = std::move(pair->second); + } else if (pair->first == "condition") { + section.conditions.emplace_back(std::move(pair->second)); + } else if (pair->first == "chainedafter") { + section.chainedafter = std::move(pair->second); + } else { + OLOG << "Unrecognized directive=" << pair->first; + return std::nullopt; + } + } break; + case ParseState::SECTION_ARGUMENTS_HEADER: + break; + case ParseState::SECTION_ARGUMENTS: { + auto pair = parseDirective(line); + if (!pair.has_value()) { + OLOG << "Invalid argument format=\"" << line + << "\" in section=" << section.name; + return std::nullopt; + } + + section.args[std::move(pair->first)] = std::move(pair->second); + } break; + + // No default to protect against any future states + } + } + + // Grab final section + ret.emplace_back(std::move(section)); + + return ret; +} + +std::unique_ptr constructIR( + const std::vector& sections) { + auto ir_root = std::make_unique(); + + // First create a handy factory map that we can use to create more detectors + std::unordered_map detectors; + for (const auto& section : sections) { + if (!section.isDetector) { + continue; + } + + // Error check that we have no duplicate detector section names + if (detectors.find(section.name) != detectors.end()) { + OLOG << "Duplicate section=" << section.name << " detected"; + return nullptr; + } + + Oomd::Config2::IR::Detector detector; + detector.name = section.plugin; + detector.args = section.args; + detectors[section.name] = std::move(detector); + } + + // Now start processing actions + std::unordered_map actions; + for (const auto& section : sections) { + if (section.isDetector) { + continue; + } + + // Error check that we have no duplicate names + if (actions.find(section.name) != actions.end()) { + OLOG << "Duplicate section=" << section.name << " detected"; + return nullptr; + } + + if (section.chainedafter.size()) { + if (actions.find(section.chainedafter) == actions.end()) { + OLOG << "Could not find section=" << section.chainedafter + << " to chain section=" << section.name << " after"; + return nullptr; + } + + Oomd::Config2::IR::Action action; + action.name = section.plugin; + action.args = section.args; + actions[section.chainedafter].acts.emplace_back(std::move(action)); + } else { + Oomd::Config2::IR::Ruleset ruleset; + ruleset.name = section.name; + + Oomd::Config2::IR::DetectorGroup dg; + dg.name = "default"; + for (const auto& cond : section.conditions) { + dg.detectors.emplace_back(detectors[cond]); // copy + } + ruleset.dgs.emplace_back(dg); + + Oomd::Config2::IR::Action action; + action.name = section.plugin; + action.args = section.args; + ruleset.acts.emplace_back(std::move(action)); + + actions[section.name] = std::move(ruleset); + } + } + + // Populate root node + for (auto& pair : actions) { + ir_root->rulesets.emplace_back(std::move(pair.second)); + } + + return ir_root; +} + +} // namespace + +namespace Oomd { +namespace Config2 { + +std::unique_ptr CompactConfigParser::parse(const std::string& input) { + auto parsed_sections = parseConfig(input); + if (!parsed_sections.has_value()) { + return nullptr; + } + + for (const auto& s : *parsed_sections) { + if (!isSectionValid(s)) { + return nullptr; + } + } + + auto ir_root = constructIR(*parsed_sections); + if (!ir_root) { + return nullptr; + } + + IR::dumpIR(*ir_root); + return ir_root; +} + +} // namespace Config2 +} // namespace Oomd diff --git a/config/CompactConfigParser.h b/config/CompactConfigParser.h new file mode 100644 index 00000000..0a21e54c --- /dev/null +++ b/config/CompactConfigParser.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018-present, Facebook, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include + +#include "oomd/config/ConfigTypes.h" + +namespace Oomd { +namespace Config2 { + +class CompactConfigParser { + public: + std::unique_ptr parse(const std::string& input); +}; + +} // namespace Config2 +} // namespace Oomd diff --git a/fixtures/oomd.rulesets b/fixtures/oomd.rulesets new file mode 100644 index 00000000..814004b5 --- /dev/null +++ b/fixtures/oomd.rulesets @@ -0,0 +1,32 @@ +[workload-stressed] +detector=pressure_above +[[args]] +cgroup=workload.slice +pressure=60 +duration=120 + +[system-tiny] +detector=memory_above +[[args]] +cgroup=system.slice +threshold=5G + +[kill-workload] +action=kill_memory_growth +condition=workload-stressed +condition=system-tiny +[[args]] +cgroup=workload.slice + +[kill-system] +action=kill_memory_growth +chainedafter=kill-workload +[[args]] +cgroup=system.slice + +[kill-user] +action=kill_memory_growth +condition=workload-stressed +condition=system-tiny +[[args]] +cgroup=user.slice diff --git a/meson.build b/meson.build index abbe26ab..b7283563 100644 --- a/meson.build +++ b/meson.build @@ -23,6 +23,7 @@ srcs = files(''' config/ConfigCompiler.cpp config/ConfigTypes.cpp config/JsonConfigParser.cpp + config/CompactConfigParser.cpp engine/DetectorGroup.cpp engine/Engine.cpp engine/Ruleset.cpp diff --git a/util/Util.cpp b/util/Util.cpp index ef1b7f12..6f653e3e 100644 --- a/util/Util.cpp +++ b/util/Util.cpp @@ -106,4 +106,13 @@ std::vector Util::split(const std::string& line, char delim) { return ret; } +void Util::strip(std::string& str) { + str.erase(0, str.find_first_not_of(kWhitespace)); + str.erase(str.find_last_not_of(kWhitespace) + 1); +} + +bool Util::isBlank(const std::string& str) { + return str.find_first_not_of(kWhitespace) == std::string::npos; +} + } // namespace Oomd diff --git a/util/Util.h b/util/Util.h index 5ae663ec..c89e7d71 100644 --- a/util/Util.h +++ b/util/Util.h @@ -25,10 +25,15 @@ namespace Oomd { class Util { public: + static constexpr auto kWhitespace = " \t\n"; + static int parseSize(const std::string& input, int64_t* output); /* Split string into tokens by delim */ static std::vector split(const std::string& line, char delim); + + static void strip(std::string& str); + static bool isBlank(const std::string& str); }; } // namespace Oomd diff --git a/util/UtilTest.cpp b/util/UtilTest.cpp index 40398ba3..b253cf3e 100644 --- a/util/UtilTest.cpp +++ b/util/UtilTest.cpp @@ -61,3 +61,26 @@ TEST(UtilTest, Split) { ASSERT_EQ(toks.size(), 1); EXPECT_EQ(toks[0], "one two three"); } + +TEST(StripTest, StripTest) { + std::string s = " 123 \t"; + std::string s1 = "\t\t 123 "; + std::string s2 = "123"; + std::string s3 = ""; + + Util::strip(s); + EXPECT_EQ(s, "123"); + Util::strip(s1); + EXPECT_EQ(s1, "123"); + Util::strip(s2); + EXPECT_EQ(s2, "123"); + Util::strip(s3); + EXPECT_EQ(s3, ""); +} + +TEST(BlankTest, BlankTest) { + EXPECT_TRUE(Util::isBlank("")); + EXPECT_TRUE(Util::isBlank(" ")); + EXPECT_FALSE(Util::isBlank(" d")); + EXPECT_TRUE(Util::isBlank(" \t\n")); +}