From b4a7121411f9288b97ad5bee542afaa66d4203aa Mon Sep 17 00:00:00 2001 From: ProgramSnail Date: Thu, 4 Aug 2022 01:22:09 +0300 Subject: [PATCH] init --- .gitignore | 14 ++---- CMakeLists.txt | 31 ++++++++++++ README.md | 9 ++++ build_graph.cpp | 110 +++++++++++++++++++++++++++++++++++++++++ build_graph.hpp | 68 +++++++++++++++++++++++++ builder.cpp | 18 +++++++ builder.hpp | 20 ++++++++ cli_common.cpp | 94 +++++++++++++++++++++++++++++++++++ cli_common.hpp | 26 ++++++++++ compile_flags.txt | 1 + dependency_manager.cpp | 44 +++++++++++++++++ dependency_manager.hpp | 29 +++++++++++ file_mode.cpp | 25 ++++++++++ file_mode.hpp | 5 ++ interactive_mode.cpp | 20 ++++++++ interactive_mode.hpp | 3 ++ main.cpp | 10 ++++ task.hpp | 12 +++++ task_queue.cpp | 30 +++++++++++ task_queue.hpp | 33 +++++++++++++ test_tasks.hpp | 13 +++++ thread_pool.cpp | 47 ++++++++++++++++++ thread_pool.hpp | 30 +++++++++++ 23 files changed, 681 insertions(+), 11 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 build_graph.cpp create mode 100644 build_graph.hpp create mode 100644 builder.cpp create mode 100644 builder.hpp create mode 100644 cli_common.cpp create mode 100644 cli_common.hpp create mode 100644 compile_flags.txt create mode 100644 dependency_manager.cpp create mode 100644 dependency_manager.hpp create mode 100644 file_mode.cpp create mode 100644 file_mode.hpp create mode 100644 interactive_mode.cpp create mode 100644 interactive_mode.hpp create mode 100644 main.cpp create mode 100644 task.hpp create mode 100644 task_queue.cpp create mode 100644 task_queue.hpp create mode 100644 test_tasks.hpp create mode 100644 thread_pool.cpp create mode 100644 thread_pool.hpp diff --git a/.gitignore b/.gitignore index 46f42f8..5162836 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,3 @@ -CMakeLists.txt.user -CMakeCache.txt -CMakeFiles -CMakeScripts -Testing -Makefile -cmake_install.cmake -install_manifest.txt -compile_commands.json -CTestTestfile.cmake -_deps +lib +bim +build diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..efba420 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.10) + +project(BuildSystem) + + +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) +set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib) + + +set(BUILD_SYSTEM_SOURCES + build_graph.cpp + builder.cpp + thread_pool.cpp + task_queue.cpp + dependency_manager.cpp + task.hpp) + +set (CLI_SOURCES + main.cpp + interactive_mode.cpp + file_mode.cpp + cli_common.cpp + test_tasks.hpp) + +add_library(BuildSystemLib ${BUILD_SYSTEM_SOURCES}) + +add_executable(BuildSystemCli ${CLI_SOURCES}) +target_link_libraries(BuildSystemCli BuildSystemLib) + +set_property(TARGET BuildSystemLib PROPERTY CXX_STANDARD 20) +set_property(TARGET BuildSystemCli PROPERTY CXX_STANDARD 20) diff --git a/README.md b/README.md new file mode 100644 index 0000000..95d9240 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +### Build System + +Сборка: + +``` +mkdir build && cd build + +cmake .. && make +``` diff --git a/build_graph.cpp b/build_graph.cpp new file mode 100644 index 0000000..65cd498 --- /dev/null +++ b/build_graph.cpp @@ -0,0 +1,110 @@ +#include "build_graph.hpp" + +#include +#include + +namespace build_system { + +const size_t BuildGraph::TraversalManager::kCurrentWayEpoch = 0; +const size_t BuildGraph::TraversalManager::kNeverVisited = 1; + +class BuildGraph::TraversalManager::View { + friend TraversalManager; + +public: + View() = delete; + + std::vector DFS(size_t vertex) { + std::vector visited; + traversal_manager.DFS(vertex, visited); + return visited; + } + + // this function saves exsisting values in "visited" array + void DFS(size_t vertex, std::vector& visited) { + traversal_manager.DFS(vertex, visited); + } + + virtual ~View() { traversal_manager.nextEpoch(); } + +private: + View(TraversalManager &traversal_manager) + : traversal_manager(traversal_manager) {} + +private: + TraversalManager &traversal_manager; +}; + +BuildGraph::TraversalManager::View BuildGraph::TraversalManager::getView() { + return View(*this); +} + +// this function saves existing values in "visited" array +void BuildGraph::TraversalManager::DFS(size_t vertex, + std::vector& current_visited) { + if (visited_[vertex] == epoch_) { + return; + } + + if (visited_[vertex] == kCurrentWayEpoch) { + throw IncorrectBuildGraph{}; + } + + visited_[vertex] = kCurrentWayEpoch; + + for (auto &dependency : build_graph_.graph_[vertex].dependences) { + DFS(dependency, current_visited); + } + + visited_[vertex] = epoch_; + + current_visited.push_back(vertex); +} + +BuildGraph::BuildGraph(const std::vector& rules, + std::vector&& tasks) + : traversal_manager_(*this) { + graph_.reserve(tasks.size()); + + for (size_t i = 0; i < tasks.size(); ++i) { + Target target; + target.task = std::forward(tasks[i]); + target.id = i; + graph_.push_back(target); + } + + for (auto &rule : rules) { + graph_[rule.target].dependences.push_back(rule.dependency); + } + + traversal_manager_.initArray(); +} + +void BuildGraph::isCorrectOrThrow(size_t target) const { + traversal_manager_.getView().DFS(target); +} + +bool BuildGraph::isCorrect(size_t target) const { + try { + isCorrectOrThrow(target); + } catch (IncorrectBuildGraph) { + return false; + } + return true; +} + +std::vector BuildGraph::getRequiredTargetsOrder(size_t target) const { + + std::vector order; + + auto rev_topsorted_required_verticles = traversal_manager_.getView().DFS(target); + + order.reserve(rev_topsorted_required_verticles.size()); + for (auto &vertex_id : rev_topsorted_required_verticles) { + order.push_back(graph_[vertex_id]); + } + + return order; +} + +} // namespace build_system \ No newline at end of file diff --git a/build_graph.hpp b/build_graph.hpp new file mode 100644 index 0000000..62e8f5f --- /dev/null +++ b/build_graph.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include +#include + +#include "task.hpp" + +namespace build_system { + +struct IncorrectBuildGraph {}; + +struct Rule { + size_t target; + size_t dependency; +}; + +struct Target { + size_t id; + Task task; + std::vector dependences; +}; + +class BuildGraph { +private: + // manage DFS, topsort, etc. safe + class TraversalManager { + private: + class View; + + public: + TraversalManager(const BuildGraph& build_graph) + : build_graph_(build_graph), epoch_(kNeverVisited + 1) {} + + void initArray() { + visited_ = std::vector(build_graph_.graph_.size(), kNeverVisited); + } + + View getView(); + + private: + void nextEpoch() { ++epoch_; } + + void DFS(size_t vertex, std::vector& current_visited); + + private: + static const size_t kCurrentWayEpoch; + static const size_t kNeverVisited; + + const BuildGraph &build_graph_; + std::vector visited_; + size_t epoch_; + }; + +public: + BuildGraph(const std::vector& rules, std::vector&& tasks); + + void isCorrectOrThrow(size_t target) const; + + bool isCorrect(size_t target) const; + + std::vector getRequiredTargetsOrder(size_t target) const; + +private: + std::vector graph_; + mutable TraversalManager traversal_manager_; +}; + +} // namespace build_system \ No newline at end of file diff --git a/builder.cpp b/builder.cpp new file mode 100644 index 0000000..e99b263 --- /dev/null +++ b/builder.cpp @@ -0,0 +1,18 @@ +#include "builder.hpp" + +namespace build_system { + +void Builder::execute(const BuildGraph& build_graph, size_t target_id) { + // can throw IncorrectBuildGraph + auto targets_order = build_graph.getRequiredTargetsOrder(target_id); + + for (auto& target : targets_order) { + thread_pool_.addTarget(std::move(target.task), target.id, target.dependences); + } + + thread_pool_.start(); + + thread_pool_.wait(); +} + +}; // namespace build_system \ No newline at end of file diff --git a/builder.hpp b/builder.hpp new file mode 100644 index 0000000..aacb848 --- /dev/null +++ b/builder.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "build_graph.hpp" +#include "thread_pool.hpp" + +namespace build_system { + +class Builder { +public: + explicit Builder(size_t num_threads) : thread_pool_(num_threads) {} + + void execute(const BuildGraph& build_graph, size_t target_id); + +private: + ThreadPool thread_pool_; +}; + +}; // namespace build_system \ No newline at end of file diff --git a/cli_common.cpp b/cli_common.cpp new file mode 100644 index 0000000..0adac29 --- /dev/null +++ b/cli_common.cpp @@ -0,0 +1,94 @@ +#include "cli_common.hpp" + +#include + +#include "test_tasks.hpp" + +void print(const std::string& message, std::ostream* out) { + if (out) { + *out << message; + } +} + +template +void getParam(const std::string& message, T ¶m, std::istream& in, std::ostream* out) { + print(message, out); + in >> param; +} + +build_system::Builder constructBuilder(std::istream& in, std::ostream* out) { + print("Builder constructor\n", out); + + size_t num_threads; + getParam("num threads: ", num_threads, in, out); + + return build_system::Builder(num_threads); +} + +build_system::BuildGraph constructBuildGraph(std::istream& in, std::ostream* out) { + print("BuildGraph constructor\n", out); + + size_t rules_count = 0; + getParam("rules count: ", rules_count, in, out); + + std::vector rules; + rules.reserve(rules_count); + + print("rules (one per line, in format \"target dependency\"):\n", out); + for (size_t i = 0; i < rules_count; ++i) { + build_system::Rule rule; + in >> rule.target >> rule.dependency; + + --rule.target; + --rule.dependency; + + rules.push_back(rule); + } + + size_t targets_count = 0; + getParam("targets count: ", targets_count, in, out); + + std::vector tasks; + tasks.reserve(targets_count); + + print("durations of tasks for targets (in seconds): ", out); + for (size_t i = 0; i < targets_count; ++i) { + size_t duration; + in >> duration; + + tasks.push_back(constructTestTask(duration)); + } + + return build_system::BuildGraph(rules, std::move(tasks)); +} + +size_t constructTargetId(std::istream& in, std::ostream* out) { + size_t target_id; + getParam("target id: ", target_id, in, out); + + --target_id; + + return target_id; +} + +void execute(build_system::Builder& builder, + build_system::BuildGraph& build_graph, size_t target_id, + std::ostream* out) { + try { + auto build_start_time = std::chrono::high_resolution_clock::now(); + + builder.execute(build_graph, target_id); + + auto build_end_time = std::chrono::high_resolution_clock::now(); + + auto build_time_ms = std::chrono::duration_cast( + build_end_time - build_start_time); + + if (out) { + *out << "overall time: " << build_time_ms.count() << "ms\n"; + } + + } catch (build_system::IncorrectBuildGraph) { + print("incorrect build graph\n", out); + } +} \ No newline at end of file diff --git a/cli_common.hpp b/cli_common.hpp new file mode 100644 index 0000000..bca8dae --- /dev/null +++ b/cli_common.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include + +#include "build_graph.hpp" +#include "builder.hpp" + +void print(const std::string& message, std::ostream* out); + +template +void getParam(const std::string& message, T& param, std::istream &in, + std::ostream* out); + +build_system::Builder constructBuilder(std::istream& in, + std::ostream* out = nullptr); + +build_system::BuildGraph constructBuildGraph(std::istream& in, + std::ostream* out = nullptr); + +size_t constructTargetId(std::istream& in, std::ostream* out = nullptr); + +void execute(build_system::Builder& builder, + build_system::BuildGraph& build_graph, size_t target_id, + std::ostream* out = nullptr); \ No newline at end of file diff --git a/compile_flags.txt b/compile_flags.txt new file mode 100644 index 0000000..e23b2ae --- /dev/null +++ b/compile_flags.txt @@ -0,0 +1 @@ +-std=c++20 diff --git a/dependency_manager.cpp b/dependency_manager.cpp new file mode 100644 index 0000000..c776b10 --- /dev/null +++ b/dependency_manager.cpp @@ -0,0 +1,44 @@ +#include "dependency_manager.hpp" + +namespace build_system { + +void DependencyManager::waitAll() { + for (size_t i = 0; i < dependency_counters_.size(); ++i) { + wait(i); + } +} + +void DependencyManager::wait(size_t id) { + auto &counter = dependency_counters_[id]; + size_t counter_value = counter.load(); + while (counter_value != 0) { + counter.wait(counter_value); + counter_value = counter.load(); + } +} + +size_t DependencyManager::add(size_t id, + const std::vector& dependences) { + size_t new_id = dependency_counters_.size(); + + dependency_counters_.emplace_back(dependences.size()); + + tasks_consequences_.emplace_back(); + + for (auto& dependency_id : dependences) { + tasks_consequences_[to_new_id[dependency_id]].push_back(new_id); + } + + to_new_id[id] = new_id; + return new_id; +} + +void DependencyManager::done(size_t id) { + for (auto &consequence_id : tasks_consequences_[id]) { + if (dependency_counters_[consequence_id].fetch_sub(1) == 1) { + dependency_counters_[consequence_id].notify_all(); + } + } +} + +}; // namespace build_system \ No newline at end of file diff --git a/dependency_manager.hpp b/dependency_manager.hpp new file mode 100644 index 0000000..ad6798b --- /dev/null +++ b/dependency_manager.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace build_system { + +class DependencyManager { +public: + DependencyManager() {} + + void waitAll(); // async + + void wait(size_t id); // async + + size_t add(size_t id, const std::vector& dependences); // sync + + void done(size_t id); // async + +private: + std::deque> dependency_counters_; + std::vector> tasks_consequences_; + std::unordered_map to_new_id; +}; + +}; // namespace build_system \ No newline at end of file diff --git a/file_mode.cpp b/file_mode.cpp new file mode 100644 index 0000000..3620c60 --- /dev/null +++ b/file_mode.cpp @@ -0,0 +1,25 @@ +#include "file_mode.hpp" + +#include +#include +#include + +#include "build_graph.hpp" +#include "builder.hpp" + +#include "cli_common.hpp" + +void fileMode(const std::string& filename) { + std::ifstream in; + in.open(filename); + + build_system::Builder builder = constructBuilder(in); + + build_system::BuildGraph build_graph = constructBuildGraph(in); + + size_t target_id = constructTargetId(in); + + execute(builder, build_graph, target_id, &std::cout); + + in.close(); +} diff --git a/file_mode.hpp b/file_mode.hpp new file mode 100644 index 0000000..839027f --- /dev/null +++ b/file_mode.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include + +void fileMode(const std::string& filename); diff --git a/interactive_mode.cpp b/interactive_mode.cpp new file mode 100644 index 0000000..ebe3296 --- /dev/null +++ b/interactive_mode.cpp @@ -0,0 +1,20 @@ +#include "interactive_mode.hpp" + +#include +#include + +#include "build_graph.hpp" +#include "builder.hpp" + +#include "cli_common.hpp" + +void interactiveMode() { + build_system::Builder builder = constructBuilder(std::cin, &std::cout); + + build_system::BuildGraph build_graph = constructBuildGraph(std::cin, &std::cout); + + size_t target_id = constructTargetId(std::cin, &std::cout); + + execute(builder, build_graph, target_id, &std::cout); + +} \ No newline at end of file diff --git a/interactive_mode.hpp b/interactive_mode.hpp new file mode 100644 index 0000000..a321906 --- /dev/null +++ b/interactive_mode.hpp @@ -0,0 +1,3 @@ +#pragma once + +void interactiveMode(); \ No newline at end of file diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..f20e8af --- /dev/null +++ b/main.cpp @@ -0,0 +1,10 @@ +#include "interactive_mode.hpp" +#include "file_mode.hpp" + +int main(int argc, char** argv) { + if (argc < 2) { + interactiveMode(); + } else { + fileMode(argv[1]); + } +} diff --git a/task.hpp b/task.hpp new file mode 100644 index 0000000..22705fb --- /dev/null +++ b/task.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace build_system { + +using Task = std::function; + +using IdentTask = std::pair; + +}; // namespace build_system \ No newline at end of file diff --git a/task_queue.cpp b/task_queue.cpp new file mode 100644 index 0000000..02007f2 --- /dev/null +++ b/task_queue.cpp @@ -0,0 +1,30 @@ +#include "task_queue.hpp" + +namespace build_system { + +bool TaskQueue::addIdentTask(IdentTask&& task) { + std::lock_guard lock(mutex_); + if (is_closed_) { + return false; + } + tasks_.push_back(std::move(task)); + task_exist_.notify_one(); + return true; +} + +std::optional TaskQueue::getIdentTask() { + std::unique_lock lock(mutex_); + while (tasks_.empty() && !is_closed_) { + task_exist_.wait(lock); + } + if (tasks_.empty()) { + return std::nullopt; + } + + IdentTask task = tasks_.front(); + tasks_.pop_front(); + + return std::move(task); +} + +}; // namespace build_system \ No newline at end of file diff --git a/task_queue.hpp b/task_queue.hpp new file mode 100644 index 0000000..2a48362 --- /dev/null +++ b/task_queue.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include + +#include "task.hpp" + +namespace build_system { + +class TaskQueue { +public: + TaskQueue() : is_closed_(false) {} + + bool addIdentTask(IdentTask&& task); + + std::optional getIdentTask(); + + void close() { + std::lock_guard lock(mutex_); + is_closed_ = true; + task_exist_.notify_all(); + } + +private: + std::deque tasks_; + bool is_closed_; + std::mutex mutex_; + std::condition_variable task_exist_; +}; + +}; // namespace build_system \ No newline at end of file diff --git a/test_tasks.hpp b/test_tasks.hpp new file mode 100644 index 0000000..28523e9 --- /dev/null +++ b/test_tasks.hpp @@ -0,0 +1,13 @@ +#include +#include +#include + +#include + +#include "task.hpp" + +inline build_system::Task constructTestTask(size_t duration) { + return [duration]() { + std::this_thread::sleep_for(std::chrono::seconds(duration)); + }; +} \ No newline at end of file diff --git a/thread_pool.cpp b/thread_pool.cpp new file mode 100644 index 0000000..04788ca --- /dev/null +++ b/thread_pool.cpp @@ -0,0 +1,47 @@ +#include "thread_pool.hpp" + +#include + +#include "task.hpp" + +namespace build_system { + +void ThreadPool::addTarget(Task&& task, size_t id, + const std::vector& dependences) { + size_t new_id = dependency_manager_.add(id, dependences); + task_queue_.addIdentTask({std::move(task), new_id}); +} + +void ThreadPool::start() { + threads_.reserve(num_threads_); + for (size_t i = 0; i < num_threads_; ++i) { + threads_.emplace_back([this]() { + while (true) { + auto may_be_task = task_queue_.getIdentTask(); + if (may_be_task == std::nullopt) { + break; + } + + dependency_manager_.wait(may_be_task.value().second); + + try { + may_be_task.value().first(); + } catch (...) { + } + + dependency_manager_.done(may_be_task.value().second); + } + }); + } +} + +void ThreadPool::wait() { + task_queue_.close(); + dependency_manager_.waitAll(); + + for (auto& thread : threads_) { + thread.join(); + } +} + +}; // namespace build_system \ No newline at end of file diff --git a/thread_pool.hpp b/thread_pool.hpp new file mode 100644 index 0000000..fe4cafa --- /dev/null +++ b/thread_pool.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include +#include + +#include "dependency_manager.hpp" +#include "task_queue.hpp" + +namespace build_system { + +class ThreadPool { +public: + explicit ThreadPool(size_t num_threads) : num_threads_(num_threads) {} + + void addTarget(Task&& task, size_t id, const std::vector& dependences); + + void start(); + + void wait(); + +private: + size_t num_threads_; + std::vector threads_; + TaskQueue task_queue_; + DependencyManager dependency_manager_; +}; + +}; // namespace build_system \ No newline at end of file