database.rb 4.48 KB
Newer Older
Valery Sizov's avatar
Valery Sizov committed
1
require 'yaml'
2
require 'open3'
Valery Sizov's avatar
Valery Sizov committed
3
4
5

module Backup
  class Database
6
    # These are the final CI tables (final prior to integration in GitLab)
7
8
9
10
11
12
    TABLES = %w{
      ci_application_settings ci_builds ci_commits ci_events ci_jobs ci_projects 
      ci_runner_projects ci_runners ci_services ci_tags ci_taggings ci_trigger_requests 
      ci_triggers ci_variables ci_web_hooks
    }

Valery Sizov's avatar
Valery Sizov committed
13
14
15
16
17
18
19
20
    attr_reader :config, :db_dir

    def initialize
      @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
      @db_dir = File.join(GitlabCi.config.backup.path, 'db')
      FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
    end

21
    def dump(mysql_to_postgresql=false)
22
23
      FileUtils.rm_f(db_file_name)
      compress_rd, compress_wr = IO.pipe
24
      compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
25
26
27
      compress_rd.close

      dump_pid = case config["adapter"]
Valery Sizov's avatar
Valery Sizov committed
28
29
      when /^mysql/ then
        $progress.print "Dumping MySQL database #{config['database']} ... "
30
31
32
        args = mysql_args
        args << '--compatible=postgresql' if mysql_to_postgresql
        spawn('mysqldump', *args, config['database'], *TABLES, out: compress_wr)
Valery Sizov's avatar
Valery Sizov committed
33
34
35
      when "postgresql" then
        $progress.print "Dumping PostgreSQL database #{config['database']} ... "
        pg_env
36
        spawn('pg_dump', '--clean', *TABLES.map { |t| "--table=#{t}" }, config['database'], out: compress_wr)
Valery Sizov's avatar
Valery Sizov committed
37
      end
38
39
40
41
      compress_wr.close

      success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? }

Valery Sizov's avatar
Valery Sizov committed
42
43
      report_success(success)
      abort 'Backup failed' unless success
44
45
46
47
48
49
50
51
52
53
54
55
56
57
      convert_to_postgresql if mysql_to_postgresql
    end

    def convert_to_postgresql
      mysql_dump_gz = db_file_name + '.mysql'
      psql_dump_gz = db_file_name + '.psql'
      drop_indexes_sql = File.join(db_dir, 'drop_indexes.sql')

      File.rename(db_file_name, mysql_dump_gz)

      $progress.print "Converting MySQL database dump to Postgres ... "
      statuses = Open3.pipeline(
        %W(gzip -cd #{mysql_dump_gz}),
        %W(python lib/support/mysql-postgresql-converter/db_converter.py - - #{drop_indexes_sql}),
58
        %W(gzip -1 -c),
59
60
61
62
63
64
65
66
67
68
69
        out: [psql_dump_gz, 'w', 0600]
      )

      if !statuses.compact.all?(&:success?)
        abort "mysql-to-postgresql-converter failed"
      end
      $progress.puts '[DONE]'.green

      $progress.print "Splicing in 'DROP INDEX' statements ... "
      statuses = Open3.pipeline(
        %W(lib/support/mysql-postgresql-converter/splice_drop_indexes #{psql_dump_gz} #{drop_indexes_sql}),
70
        %W(gzip -1 -c),
71
72
73
74
75
76
77
78
79
        out: [db_file_name, 'w', 0600]
      )
      if !statuses.compact.all?(&:success?)
        abort "Failed to splice in 'DROP INDEXES' statements"
      end

      $progress.puts '[DONE]'.green
    ensure
      FileUtils.rm_f([mysql_dump_gz, psql_dump_gz, drop_indexes_sql])
Valery Sizov's avatar
Valery Sizov committed
80
81
82
    end

    def restore
83
84
85
86
87
      decompress_rd, decompress_wr = IO.pipe
      decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
      decompress_wr.close

      restore_pid = case config["adapter"]
Valery Sizov's avatar
Valery Sizov committed
88
89
      when /^mysql/ then
        $progress.print "Restoring MySQL database #{config['database']} ... "
90
        spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
Valery Sizov's avatar
Valery Sizov committed
91
92
93
      when "postgresql" then
        $progress.print "Restoring PostgreSQL database #{config['database']} ... "
        pg_env
94
        spawn('psql', config['database'], in: decompress_rd)
Valery Sizov's avatar
Valery Sizov committed
95
      end
96
97
98
99
      decompress_rd.close

      success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }

Valery Sizov's avatar
Valery Sizov committed
100
101
102
103
104
105
106
      report_success(success)
      abort 'Restore failed' unless success
    end

    protected

    def db_file_name
107
      File.join(db_dir, 'database.sql.gz')
Valery Sizov's avatar
Valery Sizov committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
    end

    def mysql_args
      args = {
        'host'      => '--host',
        'port'      => '--port',
        'socket'    => '--socket',
        'username'  => '--user',
        'encoding'  => '--default-character-set',
        'password'  => '--password'
      }
      args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
    end

    def pg_env
      ENV['PGUSER']     = config["username"] if config["username"]
      ENV['PGHOST']     = config["host"] if config["host"]
      ENV['PGPORT']     = config["port"].to_s if config["port"]
      ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
    end

    def report_success(success)
      if success
        $progress.puts '[DONE]'.green
      else
        $progress.puts '[FAILED]'.red
      end
    end
  end
end