Merge branch 'refactor-importmetadata' into 'v2.0'

Refactor importmetadata

See merge request lemonldap-ng/lemonldap-ng!257

doc/sources/admin/renater.rst

@@ -45,6 +45,12 @@ and edit configuration:
    cp /usr/share/lemonldap-ng/bin/importMetadata /usr/share/lemonldap-ng/bin/importMetadataRenater
    vi /usr/share/lemonldap-ng/bin/importMetadataRenater
 
+.. versionchanged:: 2.0.15
+
+   Since version 2.0.15 it is no longer necessary to copy the script, you can
+   use the ``--configfile`` option to handle most customization use cases. See
+   :ref:`importmetadataconfig` below for details.
+
 Set attributes (use the SAML Name, not FriendlyName) that are provided
 by IDPs, for example:
 
@@ -92,26 +98,27 @@ Then run the script:
 
    /usr/share/lemonldap-ng/bin/importMetadataRenater -m https://metadata.federation.renater.fr/renater/main/main-idps-renater-metadata.xml -r -i "idp-renater-" -s "sp-renater-"
 
-The script provide the following options 
+The script provide the following options
 
- * -i (--idpconfprefix): Prefix used to set IDP configuration key
- * -h (--help): print this message
- * -m (--metadata): URL of metadata document
- * -s (--spconfprefix): Prefix used to set SP configuration key
- * --ignore-sp: ignore SP matching this entityID (can be specified multiple times)
- * --ignore-idp: ignore IdP matching this entityID (can be specified multiple times)
- * -a (--nagios): output statistics in Nagios format
- * -n (--dry-run): print statistics but do not apply changes
- * -v (--verbose): increase verbosity of output
- * -r (--remove): remove provider from LemonLDAP::NG if it does not appear in metadata
+* -i (--idpconfprefix): Prefix used to set IDP configuration key
+* -h (--help): print this message
+* -m (--metadata): URL of metadata document
+* -s (--spconfprefix): Prefix used to set SP configuration key
+* --ignore-sp: ignore SP matching this entityID (can be specified multiple times)
+* --ignore-idp: ignore IdP matching this entityID (can be specified multiple times)
+* -a (--nagios): output statistics in Nagios format
+* -n (--dry-run): print statistics but do not apply changes
+* -c (--configfile): use a configuration file
+* -v (--verbose): increase verbosity of output
+* -r (--remove): remove provider from LemonLDAP::NG if it does not appear in metadata
 
 
 Example :
 ::
 
     /usr/libexec/lemonldap-ng/bin/importMetadata -m https://pub.federation.renater.fr/metadata/renater/main/main-sps-renater-metadata.xml -s "sp-fed-prd" -c https://pub.federation.renater.fr/metadata/certs/renater-metadata-signing-cert-2016.pem -bs https://test-sp.federation.renater.fr -r -v -d
-    
-This command will 
+
+This command will
   * fetch all SPs metadata from renater
   * set a prefix to entity stored inside LemonLdap::NG
   * disable local modification of SP https://test-sp.federation.renater.fr
@@ -135,7 +142,7 @@ The output is the following :
 
 With "-n" options you could get a "nagios like" output with metrics :
 ::
- 
+
   /usr/libexec/lemonldap-ng/bin/importMetadataFedRenater -m https://pub.federation.renater.fr/metadata/renater/main/main-sps-renater-metadata.xml -s "sp-fed-prd" -c https://pub.federation.renater.fr/metadata/certs/renater-metadata-signing-cert-2016.pem -bs https://test-sp.federation.renater.fr -r -d -n
   Metadata loaded inside Conf: [DRY-RUN]|idp_found=0, idp_updated=0, idp_created=0, idp_removed=0, idp_rejected=0, idp_ignored=0, sp_found=1248, sp_updated=1240, sp_created=0, sp_removed=0, sp_rejected=7, sp_ignored=1
 
@@ -145,6 +152,57 @@ With "-n" options you could get a "nagios like" output with metrics :
     You need to add this in cron to refresh metadata into
     LL::NG configuration.
 
+
+.. _importmetadataconfig:
+
+Metadata import configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.0.15
+
+You can now use a configuration file for the script in order to handle most custom cases.
+
+Here is an example of a INI-formatted configuration file::
+
+    # main script options, these will be overriden by the CLI options
+    [main]
+    dry-run=1
+    verbose=1
+    metadata=http://url/to/metadata.xml
+    ; Multi-value options
+    ignore-idp=entity-id-to-ignore-1
+    ignore-idp=entity-id-to-ignore-2
+
+    # Default exported attributes for IDPs
+    [exportedAttributes]
+    cn=0;cn
+    eduPersonPrincipalName=0;eduPersonPrincipalName
+    ...
+
+    # options that apply to all providers
+    [ALL]
+    ; Disable signature requirement on requests
+    samlSPMetaDataOptionsCheckSSOMessageSignature=0
+    samlSPMetaDataOptionsCheckSLOMessageSignature=0
+    ; Store SAML assertions in session
+    samlIDPMetaDataOptionsStoreSAMLToken=1
+    ; Mark ePPN as always required
+    attribute_required_eduPersonPrincipalName=1
+    ...
+
+    # Specific provider configurations
+    [https://test-sp.federation.renater.fr]
+    ; All attributes are optional for this provider
+    attribute_required=0
+    ; Override some options
+    samlSPMetaDataOptionsNameIDFormat=persistent
+
+    [https://idp.renater.fr/idp/shibboleth]
+    ; declare an extra attribute from this provider
+    exported_attribute_eduPersonAffiliation=1;uid
+
+
+
 Add your SP into the federation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

lemonldap-ng-common/t/45-importMetadata-config.t

@@ -0,0 +1,239 @@
+#!/usr/bin/perl 
+
+use Test::More;
+use strict;
+use warnings;
+use Storable qw/dclone/;
+require_ok('./scripts/importMetadata');
+
+my $xml;
+{
+    local $/ = undef;    # Slurp mode
+    open XML, "t/data/preview-all-test-metadata.xml" or die;
+    $xml = <XML>;
+    close XML;
+}
+
+subtest 'Ignore SP' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        main => {
+            'ignore-sp' => [
+                "https://ucopia.univ-brest.fr/",
+                "http://icampus-test.univ-paris3.fr"
+            ]
+        }
+    };
+
+    # Run import
+    my ( $spCounters, $idpCounters ) =
+      transform_config( $importConf, $lmConf, $xml );
+    is( $spCounters->{created},  45 );
+    is( $spCounters->{ignored},  2 );
+    is( $idpCounters->{created}, 12 );
+    is( $idpCounters->{ignored}, 0 );
+};
+
+subtest 'Ignore IDP' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        main => {
+            'ignore-idp' => [
+                "https://serveur.uvs.sn/idp/shibboleth",
+                "https://idp-test.insa-rennes.fr/idp/shibboleth"
+            ]
+        }
+    };
+
+    # Run import
+    my ( $spCounters, $idpCounters ) =
+      transform_config( $importConf, $lmConf, $xml );
+    is( $spCounters->{created},  47 );
+    is( $spCounters->{ignored},  0 );
+    is( $idpCounters->{created}, 10 );
+    is( $idpCounters->{ignored}, 2 );
+};
+
+subtest 'Conf Prefix' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        main => {
+            'idpconfprefix' => 'renater-idp',
+            'spconfprefix'  => 'renater-sp',
+        }
+    };
+
+    # Run import
+    transform_config( $importConf, $lmConf, $xml );
+    is( scalar grep( /^renater-sp/, keys( %{ $lmConf->{samlSPMetaDataXML} } ) ),
+        47 );
+    is(
+        scalar
+          grep( /^renater-idp/, keys( %{ $lmConf->{samlIDPMetaDataXML} } ) ),
+        12
+    );
+};
+
+# Make sure matching providers who are not in the metadata are removed
+# but non-matching providers are left alone
+subtest 'Remove' => sub {
+    my $lmConf = {
+        samlSPMetaDataXML => {
+            'sp-toremove' => { samlSPMetaDataXML => "removeme" },
+            'tokeep'      => { samlSPMetaDataXML => "keepme" },
+        },
+        samlSPMetaDataExportedAttributes => {
+            'sp-toremove' => {},
+            'tokeep'      => {},
+        },
+        samlSPMetaDataOptions => {
+            'sp-toremove' => {},
+            'tokeep'      => {},
+        },
+        samlIDPMetaDataXML => {
+            'idp-toremove' => { samlSPMetaDataXML => "removeme" },
+            'tokeep'       => { samlSPMetaDataXML => "keepme" },
+        },
+        samlIDPMetaDataExportedAttributes => {
+            'idp-toremove' => {},
+            'tokeep'       => {},
+        },
+        samlIDPMetaDataOptions => {
+            'idp-toremove' => {},
+            'tokeep'       => {},
+        },
+    };
+    my $importConf = {
+        main => {
+            'remove' => 1,
+        }
+    };
+
+    # Run import
+    transform_config( $importConf, $lmConf, $xml );
+    ok( !$lmConf->{samlSPMetaDataOptions}->{'sp-toremove'} );
+    ok( $lmConf->{samlSPMetaDataOptions}->{'tokeep'} );
+    ok( !$lmConf->{samlSPMetaDataExportedAttributes}->{'sp-toremove'} );
+    ok( $lmConf->{samlSPMetaDataExportedAttributes}->{'tokeep'} );
+    ok( !$lmConf->{samlSPMetaDataXML}->{'sp-toremove'} );
+    ok( $lmConf->{samlSPMetaDataXML}->{'tokeep'} );
+    ok( !$lmConf->{samlIDPMetaDataOptions}->{'idp-toremove'} );
+    ok( $lmConf->{samlIDPMetaDataOptions}->{'tokeep'} );
+    ok( !$lmConf->{samlIDPMetaDataExportedAttributes}->{'idp-toremove'} );
+    ok( $lmConf->{samlIDPMetaDataExportedAttributes}->{'tokeep'} );
+    ok( !$lmConf->{samlIDPMetaDataXML}->{'idp-toremove'} );
+    ok( $lmConf->{samlIDPMetaDataXML}->{'tokeep'} );
+};
+
+subtest 'IDP Exported attributes' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        exportedAttributes => {
+            cn                     => '0;cn',
+            eduPersonPrincipalName => '1;eduPersonPrincipalName',
+        },
+        'https://univ-machineDebian.fr/idp/shibboleth' => {
+            exported_attribute_uid => '0;uid',
+        }
+    };
+
+    # Run import
+    transform_config( $importConf, $lmConf, $xml );
+    is_deeply(
+        $lmConf->{samlIDPMetaDataExportedAttributes}
+          ->{'idp-idp-test-insa-rennes-fr-idp-shibboleth'},
+        {
+            cn                     => '0;cn',
+            eduPersonPrincipalName => '1;eduPersonPrincipalName',
+        }
+    );
+    is_deeply(
+        $lmConf->{samlIDPMetaDataExportedAttributes}
+          ->{'idp-univ-machineDebian-fr-idp-shibboleth'},
+        {
+            cn                     => '0;cn',
+            eduPersonPrincipalName => '1;eduPersonPrincipalName',
+            uid                    => '0;uid',
+        }
+    );
+};
+
+subtest 'SP Exported attributes' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        ALL => {
+            attribute_required => 0,
+        },
+        'https://ucopia.univ-brest.fr/' => {
+            attribute_required     => 1,
+            attribute_required_uid => 0,
+        }
+    };
+
+    # Run import
+    transform_config( $importConf, $lmConf, $xml );
+    like(
+        $lmConf->{samlSPMetaDataExportedAttributes}
+          ->{'sp-umr5557-kaa-univ-lyon1-fr-sp'}->{mail},
+        qr/^0/,
+    );
+    like(
+        $lmConf->{samlSPMetaDataExportedAttributes}
+          ->{'sp-ucopia-univ-brest-fr'}->{mail},
+        qr/^1/,
+    );
+    like(
+        $lmConf->{samlSPMetaDataExportedAttributes}
+          ->{'sp-ucopia-univ-brest-fr'}->{uid},
+        qr/^0/
+    );
+};
+
+subtest 'Options' => sub {
+    my $lmConf     = {};
+    my $importConf = {
+        ALL => {
+            samlSPMetaDataOptionsCheckSSOMessageSignature => 0,
+            samlIDPMetaDataOptionsStoreSAMLToken          => 1,
+        },
+        'https://ucopia.univ-brest.fr/' => {
+            samlSPMetaDataOptionsCheckSSOMessageSignature => 1
+        },
+        'https://univ-machineDebian.fr/idp/shibboleth' => {
+            samlIDPMetaDataOptionsForceAuthn => 1,
+        },
+    };
+
+    # Run import
+    transform_config( $importConf, $lmConf, $xml );
+    is(
+        $lmConf->{samlSPMetaDataOptions}->{'sp-ucopia-univ-brest-fr'}
+          ->{samlSPMetaDataOptionsCheckSSOMessageSignature},
+        1
+    );
+    is(
+        $lmConf->{samlSPMetaDataOptions}->{'sp-wiki-uness-fr'}
+          ->{samlSPMetaDataOptionsCheckSSOMessageSignature},
+        0
+    );
+    is(
+        $lmConf->{samlIDPMetaDataOptions}
+          ->{'idp-shibboleth-2022-grenoble-archi-fr-idp'}
+          ->{samlIDPMetaDataOptionsStoreSAMLToken},
+        1
+    );
+    is(
+        $lmConf->{samlIDPMetaDataOptions}
+          ->{'idp-shibboleth-2022-grenoble-archi-fr-idp'}
+          ->{samlIDPMetaDataOptionsForceAuthn},
+        0
+    );
+    is(
+        $lmConf->{samlIDPMetaDataOptions}
+          ->{'idp-univ-machineDebian-fr-idp-shibboleth'}
+          ->{samlIDPMetaDataOptionsForceAuthn},
+        1
+    );
+};
+
+done_testing();

lemonldap-ng-common/t/45-importMetadata.t

@@ -0,0 +1,163 @@
+#!/usr/bin/perl 
+
+use Test::More;
+use strict;
+use warnings;
+use Storable qw/dclone/;
+require_ok('./scripts/importMetadata');
+
+my $xml;
+{
+    local $/ = undef;    # Slurp mode
+    open XML, "t/data/preview-all-test-metadata.xml" or die;
+    $xml = <XML>;
+    close XML;
+}
+
+my $lmConf     = {};
+my $importConf = {};
+
+# Run import
+my ( $spCounters, $idpCounters ) =
+  transform_config( $importConf, $lmConf, $xml );
+
+# Check statistics
+is_deeply(
+    $spCounters,
+    {
+        'created'  => 47,
+        'found'    => 48,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 0
+    },
+    "SP counters are expected"
+);
+is_deeply(
+    $idpCounters,
+    {
+        'created'  => 12,
+        'found'    => 13,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 0
+    },
+    "IDP counters are expected"
+);
+
+is( keys %{ $lmConf->{samlIDPMetaDataXML} }, 12,
+    "Correct amount of providers" );
+is( keys %{ $lmConf->{samlIDPMetaDataExportedAttributes} },
+    12, "Correct amount of providers" );
+is( keys %{ $lmConf->{samlIDPMetaDataOptions} },
+    12, "Correct amount of providers" );
+is( keys %{ $lmConf->{samlSPMetaDataXML} }, 47, "Correct amount of providers" );
+is( keys %{ $lmConf->{samlSPMetaDataExportedAttributes} },
+    47, "Correct amount of providers" );
+is( keys %{ $lmConf->{samlSPMetaDataOptions} },
+    47, "Correct amount of providers" );
+
+my $idp = "idp-idp-test-insa-rennes-fr-idp-shibboleth";
+my $sp  = "sp-ucopia-univ-brest-fr";
+
+is(
+    $lmConf->{samlIDPMetaDataExportedAttributes}->{$idp}
+      ->{eduPersonPrincipalName},
+    '0;eduPersonPrincipalName', "Found exported attribute"
+);
+
+is(
+    $lmConf->{samlSPMetaDataExportedAttributes}->{$sp}->{supannEtablissement},
+    join( ';',
+        0,
+        'urn:oid:1.3.6.1.4.1.7135.1.2.1.14',
+        'urn:oasis:names:tc:SAML:2.0:attrname-format:uri',
+        'supannEtablissement' ),
+    "Found optional attribute"
+);
+
+is(
+    $lmConf->{samlSPMetaDataExportedAttributes}->{$sp}->{uid},
+    join( ';',
+        1,
+        'urn:oid:0.9.2342.19200300.100.1.1',
+        'urn:oasis:names:tc:SAML:2.0:attrname-format:uri', 'uid' ),
+    "Found required attribute"
+);
+
+# Check update
+$lmConf->{samlSPMetaDataOptions}->{$sp}
+  ->{samlSPMetaDataOptionsCheckSSOMessageSignature} = 0;
+$lmConf->{samlIDPMetaDataOptions}->{$idp}
+  ->{samlIDPMetaDataOptionsAllowProxiedAuthn} = 1;
+( $spCounters, $idpCounters ) = transform_config( $importConf, $lmConf, $xml );
+
+# Check statistics
+is_deeply(
+    $spCounters,
+    {
+        'created'  => 0,
+        'found'    => 48,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 1
+    },
+    "SP counters are expected"
+);
+is_deeply(
+    $idpCounters,
+    {
+        'created'  => 0,
+        'found'    => 13,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 1
+    },
+    "IDP counters are expected"
+);
+is(
+    $lmConf->{samlSPMetaDataOptions}->{$sp}
+      ->{samlSPMetaDataOptionsCheckSSOMessageSignature},
+    1, "Configuration was updated"
+);
+is(
+    $lmConf->{samlIDPMetaDataOptions}->{$idp}
+      ->{samlIDPMetaDataOptionsAllowProxiedAuthn},
+    0, "Configuration was updated"
+);
+
+# Check idempotence
+my $oldLmConf = dclone $lmConf;
+( $spCounters, $idpCounters ) = transform_config( $importConf, $lmConf, $xml );
+
+is_deeply(
+    $spCounters,
+    {
+        'created'  => 0,
+        'found'    => 48,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 0
+    },
+    "SP counters are expected"
+);
+is_deeply(
+    $idpCounters,
+    {
+        'created'  => 0,
+        'found'    => 13,
+        'ignored'  => 0,
+        'rejected' => 1,
+        'removed'  => 0,
+        'updated'  => 0
+    },
+    "IDP counters are expected"
+);
+
+is_deeply( $lmConf, $oldLmConf );
+done_testing();