diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e606eed..62d3ee7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,11 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: [2.6, 2.7, 3.0] + ruby-version: [2.7, 3.0, 3.1, 3.2, 3.3] + postgres-version: [15, 16, 17] services: postgres: - image: postgres:13-alpine + image: postgres:${{ matrix.postgres-version }}-alpine ports: - 5432:5432 env: @@ -39,8 +40,18 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true - - name: Install Library Dependencies - run: sudo apt update && sudo apt install -y postgresql-client + - name: Install PostgreSQL ${{ matrix.postgres-version }} Client + run: | + # Add PostgreSQL APT repository + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + # Install PostgreSQL client tools + sudo apt-get install -y postgresql-client-${{ matrix.postgres-version }} + - name: Extract Test Fixtures + run: | + cd features/support + tar -xzf blog.git.tgz - name: Setup Database run: | mkdir -p config @@ -57,11 +68,15 @@ jobs: PGPORT: 5432 PGUSER: postgres PGPASSWORD: postgres - run: bundle exec rspec - # - name: Test with Cucumber - # env: - # PGHOST: localhost - # PGPORT: 5432 - # PGUSER: postgres - # PGPASSWORD: postgres - # run: bundle exec cucumber + run: | + export PATH="/usr/lib/postgresql/${{ matrix.postgres-version }}/bin:$PATH" + bundle exec rspec + - name: Test with Cucumber + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + run: | + export PATH="/usr/lib/postgresql/${{ matrix.postgres-version }}/bin:$PATH" + bundle exec cucumber diff --git a/.gitignore b/.gitignore index 879e615..47c623f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ Gemfile.lock tmp prodder-* +*.log +features/support/blog.git diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..5f6fc5e --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.3.10 diff --git a/Dockerfile b/Dockerfile index 1dc70c3..ce05134 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.0 +ARG RUBY_VERSION=3.3 FROM ruby:${RUBY_VERSION} diff --git a/Gemfile b/Gemfile index e75025d..b0556cf 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,7 @@ source 'https://rubygems.org' +gemspec + gem 'deject' gem 'thor' gem 'cocaine' @@ -10,7 +12,7 @@ group :development, :test do gem 'pry' gem 'pry-remote' gem 'rspec' - gem 'cucumber', '< 3' # FIXME - gem 'aruba', '~> 0.5.0' + gem 'cucumber' + gem 'aruba' gem 'pg' end diff --git a/README.md b/README.md index dc56778..319b523 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,111 @@ # Prodder +[![Ruby](https://img.shields.io/badge/ruby-2.7%2B-ruby.svg)](https://www.ruby-lang.org) +[![PostgreSQL](https://img.shields.io/badge/postgresql-15%2B-blue.svg)](https://www.postgresql.org) + A tool to maintain and load your Rails application's database structure, seed table contents, permissions and database settings based on its migration history and the current state in production databases. -In short: `db:reset db:migrate` - -1. Your project maintains `db/structure.sql`, `db/seeds.sql`, and optional - `db/quality_checks.sql` and `db/permissions.sql` files as it sees fit (ie, by using `prodder` as a script to dump - production and push it to your git repository). -2. Make sure `db/seeds.sql` includes the `schema_migrations` table. -3. Only new migrations will be run against prod's structure using its seed table contents. -4. Once a migration has been deployed, it should result in `db/structure.sql` and - `db/quality_checks.sql` files being modified, and any new seed data being added to - `db/seeds.sql` -- including the new entry in `schema_migrations`. -5. That migration never needs to be run in development again. Feel free to `rm`. -6. Any application related permission changes will result in `db/permissions.sql` being modified. - -## Replacing `rake db:*` -`prodder` can be included as a railtie in your application to automatically -replace many of Rails' `db:*` tasks. The only prerequisites to its usage are -the existence of `db/structure.sql`, `db/seeds.sql` with at least the -`schema_migrations` table contents included. Optional `db/quality_checks.sql` and `db/permissions.sql` -will be loaded after seeding, which can be helpful if you wish to seed the database -prior to enforcing foreign key constraints and if you want to develop in an environment -with the same permissions setup as production. +**In short:** Synchronize your development database with production structure without re-running all migrations. + +## Why Prodder? + +Traditional Rails development requires running all migrations from scratch, which: + +- ❌ Becomes slow as your migration history grows +- ❌ Can fail if old migrations are incompatible with current code +- ❌ Doesn't reflect actual production database state + +Prodder solves this by: + +- ✅ Loading production database structure directly +- ✅ Running only new migrations not yet deployed to production +- ✅ Maintaining permissions and quality checks from production +- ✅ Automatically syncing structure files from production databases + +## Requirements + +- **Ruby 2.7+** - This gem requires Ruby 2.7.0 or later +- **Bundler 2.0+** - For dependency management +- **PostgreSQL 15+** - Requires PostgreSQL 15.0 or later + +**Note:** Support for Ruby 2.6 and PostgreSQL versions older than 15 has been removed as of the latest version. If you need to use older versions, please use a previous version of this gem. + +## Overview + +Prodder follows a simple workflow: + +1. **Maintain structure files**: Your project keeps `db/structure.sql`, `db/seeds.sql`, and optionally `db/quality_checks.sql` and `db/permissions.sql` in version control. + +2. **Include migrations table**: Ensure `db/seeds.sql` includes the `schema_migrations` table from production. + +3. **Run new migrations only**: Only migrations not yet in production's `schema_migrations` table will run locally. + +4. **Update structure files**: After deploying a migration to production, update your structure files by running `prodder dump` against production. + +5. **Delete old migrations**: Once a migration is deployed and the structure files are updated, the migration file can be safely removed. + +6. **Track permissions**: Application permission changes are captured in `db/permissions.sql` for consistent development environments. + +### The Prodder Workflow + +``` +Production DB → prodder dump → db/*.sql files → Git → Development + ↓ + db:reset + new migrations +``` + +## Replacing `rake db:*` Tasks + +Prodder can be included as a Railtie in your Rails application to automatically +replace many of Rails' `db:*` tasks with versions that work with production-sourced +structure files. + +### Prerequisites + +- `db/structure.sql` - Base database structure +- `db/seeds.sql` - Seed data including `schema_migrations` table +- `db/quality_checks.sql` (optional) - Foreign keys and constraints +- `db/permissions.sql` (optional) - Database permissions for role-based access ### Installation -In your Gemfile: +Add to your Gemfile: ```ruby gem 'prodder', require: 'prodder/railtie' ``` -It doesn't really matter, but for sanity's sake, you should set your `schema_format` -to `:sql`: +Configure Rails to use SQL schema format: ```ruby # config/application.rb -module Whatever - class Application +module YourApp + class Application < Rails::Application config.active_record.schema_format = :sql end end ``` +### Basic Usage + +Once installed, use these commands: + +```bash +# Recreate database from structure and seed files +bundle exec rake db:reset + +# Run only new migrations (those not in production's schema_migrations) +bundle exec rake db:migrate + +# The typical development workflow +bundle exec rake db:reset db:migrate +``` + If you want to work with permissions setup like production: + ```ruby # config/database.yml @@ -93,49 +151,48 @@ Things that really matter: does not make sense to restore permissions in your environment if you're just going to run everything as a single, most likely superuser. - ### Details This will remove the `db:*` tasks: -* `db:_dump`: an internal task used by rails to dump the schema after migrations. Obsolete. -* `db:drop:*` -* `db:create:*` -* `db:migrate` -* `db:migrate:reset` -* `db:migrate:up` -* `db:migrate:down` -* `db:fixtures:.*` -* `db:abort_if_pending_migrations` -* `db:purge:*` -* `db:charset` -* `db:collation` -* `db:rollback` -* `db:version` -* `db:forward` -* `db:reset` -* `db:schema:*` -* `db:seed` -* `db:setup` -* `db:structure:*` -* `db:test:*` -* `test:prepare`: Rails 4.1 added this task to auto-maintain the test DB schema. +- `db:_dump`: an internal task used by rails to dump the schema after migrations. Obsolete. +- `db:drop:*` +- `db:create:*` +- `db:migrate` +- `db:migrate:reset` +- `db:migrate:up` +- `db:migrate:down` +- `db:fixtures:.*` +- `db:abort_if_pending_migrations` +- `db:purge:*` +- `db:charset` +- `db:collation` +- `db:rollback` +- `db:version` +- `db:forward` +- `db:reset` +- `db:schema:*` +- `db:seed` +- `db:setup` +- `db:structure:*` +- `db:test:*` +- `test:prepare`: Rails 4.1 added this task to auto-maintain the test DB schema. And reimplement only the following: -* `db:structure:load`: Load the contents of `db/structure.sql` into the database of your current environment. -* `db:seed`: Load `db/seeds.sql` into the database of your current environment. -* `db:quality_check`: Load `db/quality_checks.sql` into the database of your current environment, if present. -* `db:reset`: db:drop db:setup -* `db:settings`: Load the contents of `db/settings.sql` into the database of your current environment. -* `db:setup`: db:create db:structure:load db:seed db:quality_check db:settings -* `db:test:prepare`: RAILS_ENV=test db:reset db:migrate -* `db:test:clone_structure`: RAILS_ENV=test db:reset db:migrate -* `test:prepare`: db:test:prepare -* `db:drop`: Drop database as superuser -* `db:create`: Create database as `superuser` and transfer ownership to `migration_user` -* `db:migrate:*`, `db:rollback` Run migrations up/down as `migration_user` -* `db:purge:*, db:charset, db:collation, db:version, db:forward, db:rollback, db:abort_if_pending_migrations` as +- `db:structure:load`: Load the contents of `db/structure.sql` into the database of your current environment. +- `db:seed`: Load `db/seeds.sql` into the database of your current environment. +- `db:quality_check`: Load `db/quality_checks.sql` into the database of your current environment, if present. +- `db:reset`: db:drop db:setup +- `db:settings`: Load the contents of `db/settings.sql` into the database of your current environment. +- `db:setup`: db:create db:structure:load db:seed db:quality_check db:settings +- `db:test:prepare`: RAILS_ENV=test db:reset db:migrate +- `db:test:clone_structure`: RAILS_ENV=test db:reset db:migrate +- `test:prepare`: db:test:prepare +- `db:drop`: Drop database as superuser +- `db:create`: Create database as `superuser` and transfer ownership to `migration_user` +- `db:migrate:*`, `db:rollback` Run migrations up/down as `migration_user` +- `db:purge:*, db:charset, db:collation, db:version, db:forward, db:rollback, db:abort_if_pending_migrations` as appropriate users. See [lib/prodder/prodder.rake](lib/prodder/prodder.rake) @@ -144,10 +201,40 @@ for more info. This is likely to cause issues across Rails versions. No other choice really. It has been used in anger on Rails 3.2.x and Rails 4.1.x. -Confirmed working versions of Postgres: +## Development and Testing + +### Ruby Version + +This project requires Ruby 2.7+ for gem usage, though development is done on Ruby 3.3+. The development Ruby version is specified in `.ruby-version` and minimum required version in the gemspec file. + +### Testing Frameworks + +This project uses the following testing frameworks: + +- **RSpec 3.13+** for unit tests +- **Cucumber 10.x** for feature tests (upgraded from 2.x) +- **Aruba 2.x** for CLI testing (upgraded from 0.5.x) + +### Running Tests + +```bash +# Run RSpec tests +bundle exec rspec + +# Run Cucumber features +bundle exec cucumber + +# Run all tests +bundle exec rspec && bundle exec cucumber +``` + +### Supported PostgreSQL Versions + +This gem requires PostgreSQL 15.0 or later. Tested and confirmed working on: -* 9.1.11+ -* 9.2.6+ +- PostgreSQL 15.x +- PostgreSQL 16.x +- PostgreSQL 17.x ## Using prodder to maintain `db/*` files @@ -208,7 +295,7 @@ store: ### Quality Checks In some cases, such as foreign key dependencies and triggers, you may wish to defer -loading constraints on your tables until _after_ your seed data has been loaded. +loading constraints on your tables until *after* your seed data has been loaded. `prodder` treats the presence of a `quality_check_file` key in the configuration as an indication that it should split `structure_file` into those statements which create the base structure, and put the constraints into the `quality_check_file`. @@ -257,19 +344,20 @@ $ prodder push -c prodder.yml ## TODO -* Log activity as it is performed. -* Support tracking a particular branch instead of master. -* Support specifying the options to pass to each pg_dump form. -* Select dumping only a subset of a seed table. (pg_dump won't do this ...) +- Log activity as it is performed. +- Support tracking a particular branch instead of master. +- Support specifying the options to pass to each pg_dump form. +- Select dumping only a subset of a seed table. (pg_dump won't do this ...) ## Previous Contributors -* [Kyle Hargraves](https://github.com/pd) -* [Sri Rangarajan](https://github.com/Slania) -* [Emmanuel Sambo](https://github.com/esambo) -* [Cindy Wise](https://github.com/cyyyz) -* [Robert Nubel](https://github.com/rnubel) -* [Josh Cheek](https://github.com/JoshCheek) +- [Kyle Hargraves](https://github.com/pd) +- [Sri Rangarajan](https://github.com/Slania) +- [Emmanuel Sambo](https://github.com/esambo) +- [Cindy Wise](https://github.com/cyyyz) +- [Robert Nubel](https://github.com/rnubel) +- [Josh Cheek](https://github.com/JoshCheek) +- [Alexandre Castro](https://github.com/acastro2) ## License diff --git a/compose.yml b/compose.yml index 872f074..e1f9546 100644 --- a/compose.yml +++ b/compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:13-alpine + image: postgres:17-alpine volumes: - postgres_data:/var/lib/postgresql/data ports: @@ -22,7 +22,7 @@ services: context: . dockerfile: Dockerfile args: - RUBY_VERSION: 3.0 + RUBY_VERSION: 3.3 volumes: - .:/app - /app/config diff --git a/cucumber.yml b/cucumber.yml new file mode 100644 index 0000000..fea5edc --- /dev/null +++ b/cucumber.yml @@ -0,0 +1 @@ +default: --publish-quiet diff --git a/features/step_definitions/git_steps.rb b/features/step_definitions/git_steps.rb index efe631d..78fbe4b 100644 --- a/features/step_definitions/git_steps.rb +++ b/features/step_definitions/git_steps.rb @@ -1,31 +1,31 @@ -Given 'a "$project" git repository' do |project| +Given 'a {string} git repository' do |project| fixture_repo = File.join(@prodder_root, 'features', 'support', "#{project}.git") unless File.directory? fixture_repo raise "Cannot initialize repo for project #{project}; expected fixture at: #{fixture_repo}" end - run_simple "mkdir -p repos" - if File.exist? File.join(current_dir, "repos", "#{project}.git") - run_simple "chmod -R a+w repos/#{project}.git" - run_simple "rm -rf repos/#{project}.git" + run_command_and_stop "mkdir -p repos" + if File.exist? File.join(expand_path('.'), "repos", "#{project}.git") + run_command_and_stop "chmod -R a+w repos/#{project}.git" + run_command_and_stop "rm -rf repos/#{project}.git" end - run_simple "cp -pR #{fixture_repo} repos/#{project}.git" + run_command_and_stop "cp -pR #{fixture_repo} repos/#{project}.git" end -Given 'I deleted the "$project" git repository' do |project| - run_simple "rm -rf repos/#{project}.git" +Given 'I deleted the {string} git repository' do |project| + run_command_and_stop "rm -rf repos/#{project}.git" end -Given 'the "$project" git repository does not allow pushing to it' do |project| - run_simple "chmod -R a-w repos/#{project}.git" +Given 'the {string} git repository does not allow pushing to it' do |project| + run_command_and_stop "chmod -R a-w repos/#{project}.git" end -Given 'a new commit is already in the "$project" git repository' do |project| +Given 'a new commit is already in the {string} git repository' do |project| commit_to_remote project end -Then 'the new commit should be in the workspace copy of the "$project" repository' do |project| - check_file_content "prodder-workspace/#{project}/README", 'Also read this!', true +Then 'the new commit should be in the workspace copy of the {string} repository' do |project| + expect("prodder-workspace/#{project}/README").to have_file_content(/Also read this!/) end Then(/^(\d+) commits? by "([^"]+)" should be in the "([^"]+)" repository$/) do |n, author, project| @@ -35,14 +35,14 @@ end end -Then 'the file "$filename" should now be tracked' do |filename| - in_current_dir do +Then 'the file {string} should now be tracked' do |filename| + in_current_directory do git = Prodder::Git.new(File.expand_path("prodder-workspace/blog"), nil) expect(git).to be_tracked(filename) end end -Then 'the latest commit should have changed "$file" to contain "$content"' do |filename, content| +Then 'the latest commit should have changed {string} to contain {string}' do |filename, content| in_workspace('blog') do changed = `git show --name-only HEAD | grep #{filename}`.split("\n") expect(changed).to_not be_empty @@ -52,7 +52,7 @@ end end -Then 'the latest commit should not have changed "$filename"' do |filename| +Then 'the latest commit should not have changed {string}' do |filename| in_workspace('blog') do changed = `git show --name-only HEAD | grep #{filename}`.split("\n") expect(changed).to be_empty @@ -60,7 +60,7 @@ end Then 'the new commit should be in the remote repository' do - in_current_dir do + in_current_directory do latest = `git --git-dir="./repos/blog.git" log | grep prodder`.split("\n") expect(latest).to_not be_empty end diff --git a/features/step_definitions/prodder_steps.rb b/features/step_definitions/prodder_steps.rb index 7dd315c..3f3871a 100644 --- a/features/step_definitions/prodder_steps.rb +++ b/features/step_definitions/prodder_steps.rb @@ -1,27 +1,27 @@ -Given 'the "store/db/name" key is missing from "$filename"' do |filename| +Given 'the {string} key is missing from {string}' do |key, filename| # Eh, good enough! path = File.join @aruba_root, filename contents = File.read path File.open(path, 'w') { |f| f.write contents.sub(/^\s+name: \w+$/, '') } end -Given 'the "$role" role can not read from the "blog" database\'s tables' do |role| +Given 'the {string} role can not read from the "blog" database\'s tables' do |role| Prodder::PG.new.psql('prodder__blog_prod', 'REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM prodder;') end -Given 'I add an index to table "$table" on column "$column" in the "$project" project\'s database' do |table, column, project| +Given 'I add an index to table {string} on column {string} in the {string} project\'s database' do |table, column, project| Prodder::PG.new.psql "prodder__#{project}_prod", "CREATE INDEX test_index ON #{table} (#{column});" end -Given 'I add a custom parameter "$parameter" with value "$value" in the "$project" project\'s database' do |parameter, value, project| +Given 'I add a custom parameter {string} with value {string} in the {string} project\'s database' do |parameter, value, project| Prodder::PG.new.psql "prodder__#{project}_prod", "ALTER DATABASE prodder__#{project}_prod SET #{parameter} = '#{value}';" end -Given 'I add a foreign key from table "$table1" and column "$column1" to table "$table2" and column "$column2" in the "$project" project\'s database' do |table1, column1, table2, column2, project| +Given 'I add a foreign key from table {string} and column {string} to table {string} and column {string} in the {string} project\'s database' do |table1, column1, table2, column2, project| Prodder::PG.new.psql "prodder__#{project}_prod", "ALTER TABLE #{table1} ADD CONSTRAINT fk_authors FOREIGN KEY (#{column1}) REFERENCES #{table2} (#{column2});" end -Given 'no-op versions of these bins are available on my PATH: $bins' do |bins| +Given /^no-op versions of these bins are available on my PATH: (.+)$/ do |bins| paths = bins.split(/,\s*/).map { |bin| File.join(@aruba_root, "stub-#{bin}").tap do |dir| FileUtils.mkdir_p dir @@ -29,49 +29,49 @@ end }.join(File::PATH_SEPARATOR) - set_env 'PATH', "#{paths}#{File::PATH_SEPARATOR}#{ENV['PATH']}" + set_environment_variable 'PATH', "#{paths}#{File::PATH_SEPARATOR}#{ENV['PATH']}" end -Given '"$bin" is not available on my PATH' do |bin| +Given '{string} is not available on my PATH' do |bin| path = ENV['PATH'].split(File::PATH_SEPARATOR) dirs = path.select { |dir| File.exist? File.join(dir, bin) } - set_env 'PATH', path.reject { |dir| dirs.include?(dir) }.join(File::PATH_SEPARATOR) + set_environment_variable 'PATH', path.reject { |dir| dirs.include?(dir) }.join(File::PATH_SEPARATOR) end -When 'I create a new table "$table" in the "$project" database' do |table, project| +When 'I create a new table {string} in the {string} database' do |table, project| pg = Prodder::PG.new pg.psql "prodder__#{project}_prod", "CREATE TABLE #{table} ( id SERIAL PRIMARY KEY );" end -When 'I add a new author "$author" to the "$project" database' do |author, project| +When 'I add a new author {string} to the {string} database' do |author, project| pg = Prodder::PG.new pg.psql "prodder__#{project}_prod", "INSERT INTO authors (name) VALUES ('#{author}');" end -When 'I add a "$name" schema to the "$project" project\'s database' do |name, project| +When 'I add a {string} schema to the {string} project\'s database' do |name, project| pg = Prodder::PG.new pg.psql "prodder__#{project}_prod", "CREATE SCHEMA #{name} AUTHORIZATION prodder CREATE TABLE #{name}.providers ( id SERIAL PRIMARY KEY );" end -When 'I grant all permissions on table "$table" in the "$project" database to "$role"' do |table, project, role| +When 'I grant all permissions on table {string} in the {string} database to {string}' do |table, project, role| pg = Prodder::PG.new pg.psql "prodder__#{project}_prod", "GRANT ALL ON #{table} TO #{role}" end Then 'the output should contain the example config contents' do - assert_partial_output Prodder::Config.example_contents, all_output + expect(last_command_started).to have_output(an_output_string_including(Prodder::Config.example_contents)) end Then /^the workspace file "([^"]*)" should match \/([^\/]*)\/$/ do |file, partial_content| - check_file_content("prodder-workspace/#{file}", /#{partial_content}/, true) + expect("prodder-workspace/#{file}").to have_file_content(/#{partial_content}/) end Then /^the workspace file "([^"]*)" should not match \/([^\/]*)\/$/ do |file, partial_content| - check_file_content("prodder-workspace/#{file}", /#{partial_content}/, false) + expect("prodder-workspace/#{file}").not_to have_file_content(/#{partial_content}/) end Then /^the workspace file "([^"]*)" should not exist$/ do |file| - check_file_presence(["prodder-workspace/#{file}"], false) + expect("prodder-workspace/#{file}").not_to be_an_existing_file end Given(/a prodder config in "([^"]*)" with projects?: (.*)/) do |filename, projects| @@ -100,51 +100,51 @@ write_file filename, contents end -Given 'the "$project" file "$filename" contains:' do |project, filename, contents| +Given 'the {string} file {string} contains:' do |project, filename, contents| write_file "prodder-workspace/#{project}/#{filename}", contents end -Given 'the prodder config in "$filename" says to read the "$project" seed tables from "$seeds"' do |filename, project, seeds| +Given 'the prodder config in {string} says to read the {string} seed tables from {string}' do |filename, project, seeds| update_config filename do |config| config[project]['db']['tables'] = seeds end end -Given 'the prodder config in "$filename" excludes the table "$table" from the dump of "$project"' do |filename, table, project| +Given 'the prodder config in {string} excludes the table {string} from the dump of {string}' do |filename, table, project| update_config filename do |config| config[project]['db']['exclude_tables'] ||= [] config[project]['db']['exclude_tables'].push table end end -Given 'the prodder config in "$filename" excludes the schema "$schema" from the dump of "$project"' do |filename, schema, project| +Given 'the prodder config in {string} excludes the schema {string} from the dump of {string}' do |filename, schema, project| update_config filename do |config| config[project]['db']['exclude_schemas'] ||= [] config[project]['db']['exclude_schemas'].push schema end end -Given 'the prodder config in "$filename" does not include a quality check file for the "$project" project' do |filename, project| +Given 'the prodder config in {string} does not include a quality check file for the {string} project' do |filename, project| update_config filename do |config| config[project].delete 'quality_check_file' end end -Given 'the prodder config in "$filename" does not include a permissions file for the "$project" project' do |filename, project| +Given 'the prodder config in {string} does not include a permissions file for the {string} project' do |filename, project| update_config filename do |config| config[project]['permissions'].delete 'file' end end -Given 'the prodder config in "$filename" does not include permissions for the "$project" project' do |filename, project| +Given 'the prodder config in {string} does not include permissions for the {string} project' do |filename, project| update_config filename do |config| config[project].delete 'permissions' end end -Given 'the "$project" file "$filename" does not exist' do |project, filename| +Given 'the {string} file {string} does not exist' do |project, filename| begin - remove_file "prodder-workspace/#{project}/#{filename}" + remove "prodder-workspace/#{project}/#{filename}" rescue Errno::ENOENT end end diff --git a/features/support/env.rb b/features/support/env.rb index 9c623c8..e63ab40 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -1,9 +1,16 @@ require 'cucumber' require 'aruba/cucumber' -$LOAD_PATH.unshift File.expand_path('../../lib', File.dirname(__FILE__)) +$LOAD_PATH.unshift File.expand_path('../../lib', __dir__) require 'prodder' +# Configure Aruba to find the prodder executable +Aruba.configure do |config| + config.command_search_paths << File.expand_path('../../bin', __dir__) + config.exit_timeout = 10 + config.io_wait_timeout = 10 +end + module ProdderHelpers def strip_leading(string) leading = string.scan(/^\s*/).min_by &:length @@ -11,21 +18,21 @@ def strip_leading(string) end def in_workspace(name, &block) - in_current_dir { Dir.chdir("prodder-workspace/#{name}", &block) } + in_current_directory { Dir.chdir("prodder-workspace/#{name}", &block) } end def commit_to_remote(project) @dirs = ['tmp', 'aruba'] - run_simple "git clone repos/#{project}.git tmp-extra-commit-#{project}" + run_command_and_stop "git clone repos/#{project}.git tmp-extra-commit-#{project}" cd "tmp-extra-commit-#{project}" append_to_file "README", 'Also read this!' - run_simple 'git add README' - run_simple 'git commit -m "Second commit"' - run_simple "git push origin master" + run_command_and_stop 'git add README' + run_command_and_stop 'git -c user.email="test@example.com" -c user.name="Test User" commit -m "Second commit"' + run_command_and_stop "git push origin master" cd '..' - run_simple "rm -rf tmp-extra-commit-#{project}" + run_command_and_stop "rm -rf tmp-extra-commit-#{project}" end def update_config(filename, &block) @@ -90,7 +97,7 @@ def self.teardown! end def self.fixture_dbs - @fixture_dbs ||= Dir[File.join(File.dirname(__FILE__), '*.sql')].map do |sql| + @fixture_dbs ||= Dir[File.join(__dir__, '*.sql')].map do |sql| db = File.basename(sql).sub('.sql', '') [db, File.read(sql)] end @@ -100,10 +107,15 @@ def self.fixture_dbs World ProdderHelpers Before do - @prodder_root = File.expand_path('../..', File.dirname(__FILE__)) + @prodder_root = File.expand_path('../..', __dir__) @aruba_root = File.join(@prodder_root, 'tmp', 'aruba') - @aruba_timeout_seconds = 10 Dir.chdir @prodder_root + + # Configure git for tests that create commits + set_environment_variable 'GIT_AUTHOR_NAME', 'Test User' + set_environment_variable 'GIT_AUTHOR_EMAIL', 'test@example.com' + set_environment_variable 'GIT_COMMITTER_NAME', 'Test User' + set_environment_variable 'GIT_COMMITTER_EMAIL', 'test@example.com' end After('@restore-perms') do diff --git a/lib/prodder/cli.rb b/lib/prodder/cli.rb index cadc690..ec025ba 100644 --- a/lib/prodder/cli.rb +++ b/lib/prodder/cli.rb @@ -2,8 +2,6 @@ require 'thor' require 'yaml' -require 'pp' # TODO rm - class Prodder::CLI < Thor include Thor::Actions diff --git a/lib/prodder/pg.rb b/lib/prodder/pg.rb index 2d19823..710fecb 100644 --- a/lib/prodder/pg.rb +++ b/lib/prodder/pg.rb @@ -70,11 +70,12 @@ def dump_settings(db_name, filename) end end - def dump_structure(db_name, filename, options = {}) + def dump_structure(db_name, filename, **options) arguments = [ '--schema-only', '--no-privileges', '--no-owner', + '--restrict-key', 'prodder', '--host', credentials['host'], '--username', credentials['user'] ] @@ -96,6 +97,7 @@ def dump_tables(db_name, tables, filename) '--no-privileges', '--no-owner', '--disable-triggers', + '--restrict-key', 'prodder', '--host', credentials['host'], '--username', credentials['user'], *tables.map { |table| ['--table', table] }.flatten, @@ -103,12 +105,12 @@ def dump_tables(db_name, tables, filename) ] end - def dump_permissions(db_name, filename, options = {}) + def dump_permissions(db_name, filename, **options) perm_out_sql = "" user_list = [] - perm_out_sql << dump_db_access_control(db_name, user_list, options) - perm_out_sql.prepend pg_dumpall db_name, user_list, options + perm_out_sql << dump_db_access_control(db_name, user_list, **options) + perm_out_sql.prepend pg_dumpall(db_name, user_list, **options) perm_out_sql.prepend(alter_role_function) perm_out_sql.prepend(create_role_function) @@ -127,7 +129,7 @@ def dump_permissions(db_name, filename, options = {}) DEFAULT_PRIVILEGES = /^ALTER DEFAULT PRIVILEGES / SET_OBJECT_OWNERSHIP = /.* OWNER TO / - def dump_db_access_control(db_name, user_list, options) + def dump_db_access_control(db_name, user_list, **options) perm_out_sql = "" arguments = [ '--schema-only', @@ -218,7 +220,7 @@ def pg_dump(filename, cmd) end end - def pg_dumpall(db_name, user_list, options) + def pg_dumpall(db_name, user_list, **options) white_list = options[:included_users] || [] irrelevant_login_roles = irrelevant_login_roles(db_name, white_list).map { |user| user['oid'] } diff --git a/lib/prodder/version.rb b/lib/prodder/version.rb index cdfb845..082274d 100644 --- a/lib/prodder/version.rb +++ b/lib/prodder/version.rb @@ -1,3 +1,3 @@ module Prodder - VERSION = "1.8.3" + VERSION = "1.9.0" end diff --git a/prodder.gemspec b/prodder.gemspec index 12cb189..1cf1570 100644 --- a/prodder.gemspec +++ b/prodder.gemspec @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -$:.push File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(File.expand_path("lib", __dir__)) require "prodder/version" Gem::Specification.new do |s| @@ -12,11 +12,13 @@ Gem::Specification.new do |s| s.summary = "Maintain your Rails apps' structure, seed and quality_checks files using production dumps" s.description = "Migrations suck long-term. Now you can kill them routinely." - s.files = `git ls-files`.split("\n") - s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") - s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } + s.files = Dir.glob("{lib,bin,features}/**/*") + Dir.glob("*").reject { |f| File.directory?(f) } + s.test_files = Dir.glob("{test,spec,features}/**/*") + s.executables = Dir.glob("bin/*").map { |f| File.basename(f) } s.require_paths = ["lib"] + s.required_ruby_version = ">= 2.7.0" + # These dependencies do not match the Gemfile's for a reason. # These are the only dependencies necessary to satisfy inclusion of this # gem in a Rails application; any dependencies necessary to run prodder