Tag Archives: CEPH

We use a CEPH storage solution and specifically want to use the Ceph Object Gateway with the S3 API through a Java client. The API is based on the AWS S3 standard however requires some special tweaking to work. Took me some effort to get a working connection, so here to share:

<dependency>
  <groupId>com.amazonaws</groupId>
  <artifactId>aws-java-sdk</artifactId>
  <version>1.11.325</version>
</dependency>

We can use either the new AmazonS3ClientBuilder

package nl.rubix.s3;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.SDKGlobalConfiguration;

public class AmazonS3ClientBuilder
{
  public static void main(String[] args)
  {
    String accessKey = "XXXXX";
    String secretKey = "XXXXX";

    // Our firewall on DEV does some weird stuff so we disable SSL cert check
    System.setProperty(SDKGlobalConfiguration.DISABLE_CERT_CHECKING_SYSTEM_PROPERTY,"true");
    if (SDKGlobalConfiguration.isCertCheckingDisabled())
    {
      System.out.println("Cert checking is disabled");
    }
		
    // S3 Client configuration
    ClientConfiguration config = new ClientConfiguration();
    // Not the standard "AWS3SignerType", maar expliciet signerTypeV2
    config.setSignerOverride("S3SignerType");
    config.setProtocol(Protocol.HTTPS);
    config.setProxyHost("proxy.rubix.nl");

    config.setProxyPort(8080);
    // S3 Credentials
    BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
    // S3 Endpoint
    AwsClientBuilder.EndpointConfiguration endpointConfiguration = new
      AwsClientBuilder.EndpointConfiguration("objects.dc1.rubix.nl", "");
    AmazonS3 s3 = com.amazonaws.services.s3.AmazonS3ClientBuilder.standard()
      .withClientConfiguration(config)
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(endpointConfiguration)
      .build();
    
    System.out.println("===========================================");
    System.out.println(" Connection to the Rubix S3 ");
    System.out.println("===========================================\n");
    try { 
       /*
       * List of buckets and objects in our account
       */
       System.out.println("Listing buckets and objects");
       for (Bucket bucket : s3.listBuckets())
       {
         System.out.println(" - " + bucket.getName() +" "
           + "(owner = " + bucket.getOwner()
           + " "
           + "(creationDate = " + bucket.getCreationDate());
         ObjectListing objectListing = s3.listObjects(new ListObjectsRequest()
           .withBucketName(bucket.getName()));
         for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) 
         {
           System.out.println(" --- " + objectSummary.getKey() +" "
           + "(size = " + objectSummary.getSize() + ")" +" "
           + "(eTag = " + objectSummary.getETag() + ")");
           System.out.println();
         }
       }
     }
     catch (AmazonServiceException ase)
     {
       System.out.println("Caught an AmazonServiceException, which means your request made it to S3, but was rejected with an error response for some reason.");
       System.out.println("Error Message:    " + ase.getMessage());
       System.out.println("HTTP Status Code: " + ase.getStatusCode());
       System.out.println("AWS Error Code: " + ase.getErrorCode());
       System.out.println("Error Type: " + ase.getErrorType());
       System.out.println("Request ID: " + ase.getRequestId());
     }
     catch (AmazonClientException ace)
     {
       System.out.println("Caught an AmazonClientException, which means the client encountered "
       + "a serious internal problem while trying to communicate with S3,
       + "such as not being able to access the network.");
       System.out.println("Error Message: " + ace.getMessage());
     }

or make it work with the older and depricated AmazonS3Client

package nl.rubix.s3;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.SDKGlobalConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amaonaws.services.s3.model.Bucket;

public class BasicAWSCredentials
{
    public static void main(String[] args)
    {
        String accessKey = "XXXXXXX";
        String secretKey = "XXXXXXX";
        System.setProperty(SDKGlobalConfiguration.DISABLE_CERT_CHECKING_SYSTEM_PROPERTY,"true"
    );

    if (SDKGlobalConfiguration.isCertCheckingDisabled())
    {
        System.out.println("Cert checking is disabled");
    }
    AWSCredentials credentials = new com.amazonaws.auth.BasicAWSCredentials(accessKey,secretKey);

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setSignerOverride("S3SignerType");
    clientConfig.setProxyHost("proxy.rubix.nl");
    clientConfig.setProxyPort(8080);

    AmazonS3 conn = new AmazonS3Client(credentials, clientConfig);
    conn.setEndpoint("objects.gn3.rubix.nl");

    for (Bucket bucket : conn.listBuckets())
    {
      System.out.println(" - " + bucket.getName() 
        + " "
        + "(owner = " + bucket.getOwner()
        + " "
        + "(creationDate = " + bucket.getCreationDate());
    }
  }
}

Hope it helps!