Skip to content

RANGER-5205: Liquibase based Zero-Downtime-Upgrade framework for Ranger #889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sanket-Shelar wants to merge 1 commit into
base: master
Choose a base branch
from
Open

RANGER-5205: Liquibase based Zero-Downtime-Upgrade framework for Ranger #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions distro/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,7 @@
<descriptor>src/main/assembly/plugin-presto.xml</descriptor>
<descriptor>src/main/assembly/plugin-trino.xml</descriptor>
<descriptor>src/main/assembly/sample-client.xml</descriptor>
<descriptor>src/main/assembly/liquibase-upgrade.xml</descriptor>
</descriptors>
</configuration>
</execution>
Expand Down Expand Up @@ -1090,6 +1091,7 @@
<descriptor>src/main/assembly/plugin-presto.xml</descriptor>
<descriptor>src/main/assembly/plugin-trino.xml</descriptor>
<descriptor>src/main/assembly/sample-client.xml</descriptor>
<descriptor>src/main/assembly/liquibase-upgrade.xml</descriptor>
</descriptors>
</configuration>
</execution>
Expand Down
77 changes: 77 additions & 0 deletions distro/src/main/assembly/liquibase-upgrade.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<assembly>
<id>liquibase-upgrade</id>
<formats>
<format>tar.gz</format>
</formats>
<baseDirectory>${project.parent.name}-${project.version}-liquibase-upgrade</baseDirectory>
<includeBaseDirectory>true</includeBaseDirectory>
<moduleSets>
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<include>org.apache.ranger:ranger-liquibase-upgrade:jar:${project.version}</include>
</includes>
<binaries>
<outputDirectory>lib</outputDirectory>
<includeDependencies>true</includeDependencies>
<unpack>false</unpack>
<directoryMode>755</directoryMode>
<fileMode>644</fileMode>
<includes>
<include>org.liquibase:liquibase-core:jar:${liquibase.version}</include>
<include>com.beust:jcommander:jar:${jcommander.version}</include>
<include>org.slf4j:slf4j-api:jar:${slf4j.version}</include>
<!-- for liquibase-->
<include>org.slf4j:jul-to-slf4j:jar:${slf4j.version}</include>
<!-- spring related dependencies-->
<include>org.springframework:spring-context:jar:${springframework.version}</include>
<include>org.springframework:spring-beans:jar:${springframework.version}</include>
<include>org.springframework:spring-core:jar:${springframework.version}</include>
<include>org.springframework:spring-aop:jar:${springframework.version}</include>
<include>org.springframework:spring-expression:jar:${springframework.version}</include>
<!-- <include>org.yaml:snakeyaml:jar:${snakeyaml.version}</include> -->
</includes>
</binaries>
</moduleSet>
</moduleSets>
<fileSets>
<fileSet>
<directory>${project.parent.basedir}/liquibase-database-upgrade/target/scripts</directory>
<outputDirectory></outputDirectory>
<includes>
<include>*.sh</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/liquibase-database-upgrade/target/changelogs</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>**</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/liquibase-database-upgrade/target/classes</directory>
<outputDirectory>sample_resources</outputDirectory>
<includes>
<include>*.properties</include>
</includes>
</fileSet>
</fileSets>
</assembly>
5 changes: 4 additions & 1 deletion kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path";
private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias";
private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias";
private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password";
public static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password";
private static final String HSM_ENABLED = "ranger.ks.hsm.enabled";
private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias";
private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password";
Expand All @@ -81,6 +81,9 @@ public class RangerKeyStoreProvider extends KeyProvider {
private static final String AZURE_CLIENT_SECRET_ALIAS = "ranger.kms.azure.client.secret.alias";
private static final String AZURE_CLIENT_SECRET = "ranger.kms.azure.client.secret";
private static final String AWS_KMS_ENABLED = "ranger.kms.awskms.enabled";
public static final String DB_URL = "ranger.ks.jpa.jdbc.url";
public static final String DB_USER = "ranger.ks.jpa.jdbc.user";
public static final String DB_DRIVER = "ranger.ks.jpa.jdbc.driver";
private static final String AWS_CLIENT_SECRETKEY_ALIAS = RangerAWSKMSProvider.AWS_CLIENT_SECRETKEY + ".alias";
private static final String AWS_CLIENT_SECRETKEY = RangerAWSKMSProvider.AWS_CLIENT_SECRETKEY;
private static final String TENCENT_KMS_ENABLED = "ranger.kms.tencentkms.enabled";
Expand Down
29 changes: 29 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-finalize.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:pro="http://www.liquibase.org/xml/ns/pro"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">

<!-- For every version upgrade add one master finalize xml file here. -->
<!-- TODO: Add the below statement before delivering to customers-->
<include file="kms/db/test_3.x/master-finalize-test_3.x.xml"/>

</databaseChangeLog>
30 changes: 30 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-upgrade.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:pro="http://www.liquibase.org/xml/ns/pro"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">

<!-- For every version upgrade add one master upgrade xml file here. -->
<!-- TODO: Add the below statement before delivering to customers-->
<include file="kms/db/test_3.x/master-upgrade-test_3.x.xml"/>


</databaseChangeLog>
42 changes: 42 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/postgres/create_trigger_kmsencoded_postgres.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

--liquibase formatted sql

-- Create a trigger function
CREATE OR REPLACE FUNCTION copy_data_trigger_function()
RETURNS TRIGGER AS
'
BEGIN
-- If the legacy column has data but the new column does not, sync forward
IF NEW.kms_encoded IS NOT NULL AND NEW.kms_encoded_value IS NULL THEN
NEW.kms_encoded_value := NEW.kms_encoded;

-- If the new column has data but the legacy column does not, sync backward
ELSIF NEW.kms_encoded_value IS NOT NULL AND NEW.kms_encoded IS NULL THEN
NEW.kms_encoded := NEW.kms_encoded_value;

END IF;

RETURN NEW;
END;
'
LANGUAGE plpgsql;

-- Create a trigger
CREATE TRIGGER copy_data_trigger
BEFORE INSERT OR UPDATE ON ranger_keystore
FOR EACH ROW
EXECUTE PROCEDURE copy_data_trigger_function();
41 changes: 41 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_column.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="cleanup_5a" author="Sailaja" failOnError="false">
<preConditions>
<and>
<!-- Ensure here that important steps were completed-->
<columnExists tableName="ranger_keystore"
columnName="kms_encoded_value" />
<changeSetExecuted id="5a" author="fateh" changeLogFile="kms/db/test_3.x/kms_rename_col_zdu.xml"/>
<changeSetExecuted id="5b" author="fateh" changeLogFile="kms/db/test_3.x/kms_rename_col_zdu.xml"/>
</and>
</preConditions>
<!--
Delete triggers/functions etc if required
Drop constraints/indexes that reference the old column (if any)
<sql>ALTER TABLE ranger_keystore DROP CONSTRAINT constraint_name;</sql> -->
<!-- Drop the old column -->
<dropColumn tableName="ranger_keystore" columnName="kms_encoded"/>
</changeSet>
</databaseChangeLog>
33 changes: 33 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_trigger.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:pro="http://www.liquibase.org/xml/ns/pro"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">

<changeSet id="cleanup_5c" author="fateh">
<sql dbms="postgresql" >
DROP TRIGGER IF EXISTS copy_data_trigger ON ranger_keystore;
</sql>
<rollback>empty</rollback>
</changeSet>

</databaseChangeLog>
27 changes: 27 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/test_3.x/db_javapatches.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">

<changeSet id="6" author="Sailaja" runAlways="false" failOnError="true">
<customChange class="org.apache.ranger.db.upgrade.patches.kms.UpdateKMSAttrs">
</customChange>
</changeSet>
</databaseChangeLog>
50 changes: 50 additions & 0 deletions liquibase-database-upgrade/changelogs/kms/db/test_3.x/kms_rename_col_zdu.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:pro="http://www.liquibase.org/xml/ns/pro"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">
<changeSet id="5a" author="fateh">
<addColumn tableName="ranger_keystore">
<column name="kms_encoded_value"
type="varchar(2048)"/>
</addColumn>
</changeSet>

<changeSet id="5b" author="fateh">
<sql dbms="postgresql" >
update ranger_keystore set kms_encoded_value = kms_encoded;
</sql>
<rollback>empty</rollback>
</changeSet>


<changeSet id="5c" author="fateh">
<sqlFile path="kms/db/postgres/create_trigger_kmsencoded_postgres.sql" dbms="postgresql" />
<rollback>
<sql dbms="postgresql">
DROP TRIGGER IF EXISTS copy_data_trigger ON ranger_keystore;
</sql>
</rollback>
</changeSet>

</databaseChangeLog>
Loading