Google drive api 提高每个用户的Google驱动器API限制并不能防止速率限制异常

Google drive api 提高每个用户的Google驱动器API限制并不能防止速率限制异常,google-drive-api,google-oauth,Google Drive Api,Google Oauth,我正在使用一个进程,该进程使用driveapi将纯文本文件上传到googledrive。即使请求的实际数量远未达到API控制台中设置的驱动器API的每个用户限制,该进程也经常遇到速率限制异常。事实上,设置每个用户的限制似乎不会影响我们接收异常的速率。是否存在其他限制(除每用户限制外)来控制每秒可以发出多少请求?可以调整吗 该过程对这些异常使用指数回退,因此操作最终是成功的。我们每秒只发出5个左右的请求,每个用户的限制设置为100 Caused by: com.google.api.client.

我正在使用一个进程,该进程使用driveapi将纯文本文件上传到googledrive。即使请求的实际数量远未达到API控制台中设置的驱动器API的每个用户限制,该进程也经常遇到速率限制异常。事实上,设置每个用户的限制似乎不会影响我们接收异常的速率。是否存在其他限制(除每用户限制外)来控制每秒可以发出多少请求?可以调整吗

该过程对这些异常使用指数回退,因此操作最终是成功的。我们每秒只发出5个左右的请求,每个用户的限制设置为100

Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
  "code" : 403,
  "errors" : [ {
    "domain" : "usageLimits",
    "message" : "Rate Limit Exceeded",
    "reason" : "rateLimitExceeded"
  } ],
  "message" : "Rate Limit Exceeded"
}
编辑:这是开发人员提供的代码的“简化”版本。我们正在使用具有域委派的服务帐户,如所述:

package com.seto.fs.daemon;
导入java.io.File;
导入java.io.FileOutputStream;
导入java.io.IOException;
导入java.util.ArrayList;
导入java.util.array;
导入java.util.Collections;
导入java.util.List;
导入java.util.concurrent.AtomicInteger;
导入com.google.api.client.auth.oauth2.Credential;
导入com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
导入com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
导入com.google.api.client.googleapis.json.GoogleJsonResponseException;
导入com.google.api.client.http.FileContent;
导入com.google.api.client.http.httpbackOfficeExceptionHandler;
导入com.google.api.client.http.httpbackoff未成功响应Handler;
导入com.google.api.client.http.httpbackoff失败响应handler.BackOffRequired;
导入com.google.api.client.http.HttpRequest;
导入com.google.api.client.http.HttpRequestInitializer;
导入com.google.api.client.http.HttpResponse;
导入com.google.api.client.http.HttpTransport;
导入com.google.api.client.json.JsonFactory;
导入com.google.api.client.json.jackson2.JacksonFactory;
导入com.google.api.client.testing.util.MockBackOff;
导入com.google.api.client.util.DateTime;
导入com.google.api.client.util.ExponentialBackOff;
导入com.google.api.services.drive.drive;
导入com.google.api.services.drive.drive.Files.Insert;
导入com.google.api.services.drive.DriveScopes;
导入com.google.api.services.drive.model.ChildList;
导入com.google.api.services.drive.model.ChildReference;
导入com.google.api.services.drive.model.File.Labels;
导入com.google.api.services.drive.model.ParentReference;
公开课考试{
私有静态最终int testFilesCount=100;
私有静态最终int-threadscont=3;
私有静态最终AtomicInteger rateLimitErrorsCount=新的AtomicInteger(0);
私有静态最终字符串impersonatedUser=“”;
私有静态最终字符串serviceAccountID=“@developer.gserviceaccount.com”;
私有静态最终字符串serviceAccountPK=“/path/to/-privatekey.p12”;
公共静态void main(字符串[]args)引发异常{
//创建HTTP传输
HttpTransport HttpTransport=GoogleNetHttpTransport.newTrustedTransport();
//创建JsonFactory
最终JsonFactory JsonFactory=新JacksonFactory();
//为服务帐户创建Google凭据
最终凭证=新的GoogleCredential.Builder()
.setTransport(httpTransport)
.setJsonFactory(jsonFactory)
.setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
.setServiceAccountUser(模拟用户)
.setServiceAccountId(serviceAccountID)
.SetServiceAccountPrivateKeyFromp12文件(新文件(serviceAccountPK))
.build();
//创建驱动器客户端
终传动驱动器=new Drive.Builder(httpTransport、jsonFactory、new HttpRequestInitializer(){
公共无效初始化(HttpRequest请求)引发IOException{
请求。设置ContentLoggingLimit(0);
请求。setCurlLoggingEnabled(false);
//授权初始化
初始化(请求);
//5xx响应和403速率限制超出错误的指数回退
HttpBackOffUnsuccessfulResponseHandler服务器错误处理程序
=新的HttpBackOffUnsuccessfulResponseHandler(新的ExponentialBackOff.Builder().build());
serverErrorHandler.setBackOffRequired(需要新的BackOffRequired(){
需要公共布尔值(HttpResponse响应){
返回响应。getStatusCode()/100==5
||(response.getStatusCode()==403&&isRateLimitExceeded(
谷歌JSONResponseException.from(jsonFactory,response));
}
});
setUnsuccessfulResponseHandler(serverErrorHandler);
//因套接字连接错误而退出
MockBackOff backOff=新的MockBackOff();
退避.退避.米利斯(2000年);
退避。设置最大值(5);
setIOExceptionHandler(新的HttpBackOfficeExceptionHandler(backOff));
}
}).setApplicationName(“GoogleDriveUploadFile/1.0”).build();
//获取根文件夹id
最终字符串rootFolderId=drive.about().get().execute().getRootFolderId();
//查询根文件夹下的所有子项
ChildList结果=drive.children().list(rootFolderId.execute();
//删除根文件夹下的所有子文件夹
for(ChildReference子项:result.getItems()){
System.out.println(“删除子项:+child.getId());
drive.files().delete(child.getId()).execute();
}
//创建驱动器文件夹
com.google.api.services.drive.model.File folderMetadata
=新建com.google.api.service
package com.seto.fs.daemon;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpBackOffIOExceptionHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler.BackOffRequired;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.util.MockBackOff;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files.Insert;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File.Labels;
import com.google.api.services.drive.model.ParentReference;


public class Test {
    private static final int testFilesCount = 100;
    private static final int threadsCount = 3;
    private static final AtomicInteger rateLimitErrorsCount = new AtomicInteger(0);

    private static final String impersonatedUser = "<impersonatedUserEmail>";
    private static final String serviceAccountID = "<some-id>@developer.gserviceaccount.com";
    private static final String serviceAccountPK = "/path/to/<public_key_fingerprint>-privatekey.p12";

    public static void main(String[] args) throws Exception {
        // Create HTTP transport
        HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        // Create JsonFactory
        final JsonFactory jsonFactory = new JacksonFactory();

        // Create Google credential for service account
        final Credential credential = new GoogleCredential.Builder()
                .setTransport(httpTransport)
                .setJsonFactory(jsonFactory)
                .setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
                .setServiceAccountUser(impersonatedUser)
                .setServiceAccountId(serviceAccountID)
                .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountPK))
                .build();

        // Create Drive client
        final Drive drive = new Drive.Builder(httpTransport, jsonFactory, new HttpRequestInitializer() {
            public void initialize(HttpRequest request) throws IOException {
                request.setContentLoggingLimit(0);
                request.setCurlLoggingEnabled(false);

                // Authorization initialization
                credential.initialize(request);

                // Exponential Back-off for 5xx response and 403 rate limit exceeded error
                HttpBackOffUnsuccessfulResponseHandler serverErrorHandler
                    = new HttpBackOffUnsuccessfulResponseHandler(new ExponentialBackOff.Builder().build());
                serverErrorHandler.setBackOffRequired(new BackOffRequired() {
                    public boolean isRequired(HttpResponse response) {
                        return response.getStatusCode() / 100 == 5
                            || (response.getStatusCode() == 403 && isRateLimitExceeded(
                                    GoogleJsonResponseException.from(jsonFactory, response)));
                    }
                });
                request.setUnsuccessfulResponseHandler(serverErrorHandler);

                // Back-off for socket connection error
                MockBackOff backOff = new MockBackOff();
                backOff.setBackOffMillis(2000);
                backOff.setMaxTries(5);
                request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(backOff));
            }
        }).setApplicationName("GoogleDriveUploadFile/1.0").build();

        // Get root folder id
        final String rootFolderId = drive.about().get().execute().getRootFolderId();

        // Query all children under root folder
        ChildList result = drive.children().list(rootFolderId).execute();

        // Delete all children under root folder
        for (ChildReference child : result.getItems()) {
            System.out.println("Delete child: " + child.getId());
            drive.files().delete(child.getId()).execute();
        }

        // Create a drive folder
        com.google.api.services.drive.model.File folderMetadata
            = new com.google.api.services.drive.model.File();
        folderMetadata.setMimeType("application/vnd.google-apps.folder")
            .setParents(Arrays.asList(new ParentReference().setId(rootFolderId)))
            .setTitle("DriveTestFolder");
        final com.google.api.services.drive.model.File driveTestFolder = drive.files().insert(folderMetadata).execute();

        // Create test files
        final List<File> testFiles = Collections.synchronizedList(createTestFiles());

        // Run threads to upload files to drive
        List<Thread> threads = new ArrayList<Thread>();
        for (int i = 0; i < threadsCount; i++) {
            Thread thread = new Thread(new Runnable() {
                public void run() {
                    while (testFiles.size() > 0) {
                        try {
                            File testFile = testFiles.remove(0);

                            // The file meta data
                            com.google.api.services.drive.model.File fileMetadata =
                                new com.google.api.services.drive.model.File()
                                .setTitle(testFile.getName()).setParents(Arrays.asList(new ParentReference().setId(driveTestFolder.getId())))
                                .setLabels(new Labels().setRestricted(false)).setMimeType("text/plain")
                                .setModifiedDate(new DateTime(testFile.lastModified()))
                                .setDescription("folder:MyDrive " + testFile.getName());

                            // Insert to drive
                            FileContent fileContent = new FileContent("text/plain", testFile);
                            Insert insertFileCommand = drive.files().insert(fileMetadata, fileContent)
                                    .setUseContentAsIndexableText(true);
                            insertFileCommand.getMediaHttpUploader().setDirectUploadEnabled(true);

                            insertFileCommand.execute();

                            System.out.println(testFile.getName() + " is uploaded");
                        } catch (IOException e) {
                            e.printStackTrace();
                        } catch (IndexOutOfBoundsException e) {
                            // ignore
                        }
                    }
                }
            });
            threads.add(thread);
        }

        long startTime = System.currentTimeMillis();

        for (Thread thread : threads) {
            thread.start();
        }

        for (Thread thread : threads) {
            thread.join();
        }

        System.out.println("Total time spent: " + (System.currentTimeMillis() - startTime)
                + "ms for " + testFilesCount + " files with " + threadsCount + " threads");
        System.out.println("Rate limit errors hit: " + rateLimitErrorsCount.intValue());
    }

    private static List<File> createTestFiles() throws Exception {

        // Create test files directory
        File testFolder = new File("TestFiles");
        testFolder.mkdir();

        // Create test files
        List<File> testFiles = new ArrayList<File>();
        for (int i = 0; i < testFilesCount; i++) {
            File testFile = new File("TestFiles/" + i + ".txt");
            FileOutputStream fops = new FileOutputStream(testFile);
            fops.write(testFile.getAbsolutePath().getBytes());
            fops.close();
            testFiles.add(testFile);
        }
        return testFiles;
    }

    private static boolean isRateLimitExceeded(GoogleJsonResponseException ex) {
        boolean result = false;
        if (ex.getDetails() != null && ex.getDetails().getErrors() != null
                && ex.getDetails().getErrors().size() > 0) {
            String reason = ex.getDetails().getErrors().get(0).getReason();
            result = "rateLimitExceeded".equals(reason) || "userRateLimitExceeded".equals(reason);
            if (result) {
                rateLimitErrorsCount.incrementAndGet();
                System.err.println("Rate limit error");
            }
        }

        return result;
    }

}