Dear ALL:
bin/start.sh will error when SciHubDataSource.Sentinel1/Sentinel2.scope=3 or SciHubDataSource.Sentinel1/Sentinel2.scope=2
bin/start.sh will ok when is SciHubDataSource.Sentinel1.scope=1 and SciHubDataSource.Sentinel2.scope=1
and
I want query & download from SciHubDataSource .
config/sen2agri-services.properties
//## Data sources default purposes
//## 1 = query, 2 = download, 3 = query & download
SciHubDataSource.Sentinel1.scope=3
SciHubDataSource.Sentinel2.scope=3
#AWSDataSource.Sentinel2.scope=3
//## If the local_archive_path is set, the specified local repository will be used for downloading products
//## AWSDataSource.Sentinel2.local_archive_path=/mnt/archive/test/Local_Store_Test/local_repository
//## Possible values for fetch_mode
//## 1 = OVERWRITE, 2 = RESUME, 3 = COPY, 4 = SYMLINK
//## 3 or 4 to be set if local_archive_path is set to a repository like /eodata/…
AWSDataSource.Sentinel2.fetch_mode=1
USGSDataSource.Landsat8.scope=3
AWSDataSource.Landsat8.scope=2
SciHubDataSource.Sentinel1.enabled=true
SciHubDataSource.Sentinel2.enabled=true
AWSDataSource.Sentinel2.enabled=true
USGSDataSource.Landsat8.enabled=true
AWSDataSource.Landsat8.enabled=false
AWSDataSource.Sentinel2.usePadding=true
AWSDataSource.Landsat8.usePadding=true
##star.sh Error
2018-08-26 18:13:38.271 WARN 19386 — [ main] ationConfigEmbeddedWebApplicationContext : Exception encountered during context initialization - cancelling refresh attempt: org.springframework.beans.factory.UnsatisfiedDependencyException: Error creating bean with name ‘siteService’: Unsatisfied dependency expressed through field ‘scheduleManager’; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name ‘scheduleManager’: Invocation of init method failed; nested exception is java.lang.IllegalStateException: Duplicate key org.esa.sen2agri.entities.DataSourceConfiguration@4b7f29e
2018-08-26 18:13:38.273 INFO 19386 — [ main] j.LocalContainerEntityManagerFactoryBean : Closing JPA EntityManagerFactory for persistence unit ‘sen2agri’
2018-08-26 18:13:38.279 INFO 19386 — [ main] utoConfigurationReportLoggingInitializer :
Error starting ApplicationContext. To display the auto-configuration report re-run your application with ‘debug’ enabled.
2018-08-26 18:13:38.287 ERROR 19386 — [ main] o.s.boot.SpringApplication : Application startup failed
org.springframework.beans.factory.UnsatisfiedDependencyException: Error creating bean with name ‘siteService’: Unsatisfied dependency expressed through field ‘scheduleManager’; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name ‘scheduleManager’: Invocation of init method failed; nested exception is java.lang.IllegalStateException: Duplicate key org.esa.sen2agri.entities.DataSourceConfiguration@4b7f29e
at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor$AutowiredFieldElement.inject(AutowiredAnnotationBeanPostProcessor.java:588) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.annotation.InjectionMetadata.inject(InjectionMetadata.java:88) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor.postProcessPropertyValues(AutowiredAnnotationBeanPostProcessor.java:366) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.populateBean(AbstractAutowireCapableBeanFactory.java:1264) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:553) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:761) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:867) ~[spring-context-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543) ~[spring-context-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.boot.context.embedded.EmbeddedWebApplicationContext.refresh(EmbeddedWebApplicationContext.java:122) ~[spring-boot-1.5.7.RELEASE.jar:1.5.7.RELEASE]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:693) [spring-boot-1.5.7.RELEASE.jar:1.5.7.RELEASE]
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:360) [spring-boot-1.5.7.RELEASE.jar:1.5.7.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:303) [spring-boot-1.5.7.RELEASE.jar:1.5.7.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1118) [spring-boot-1.5.7.RELEASE.jar:1.5.7.RELEASE]
at org.esa.sen2agri.ServicesStartup.main(ServicesStartup.java:42) [sen2agri-startup-1.9-SNAPSHOT.jar:na]
Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name ‘scheduleManager’: Invocation of init method failed; nested exception is java.lang.IllegalStateException: Duplicate key org.esa.sen2agri.entities.DataSourceConfiguration@4b7f29e
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:137) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:409) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1620) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:555) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:208) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1138) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1066) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor$AutowiredFieldElement.inject(AutowiredAnnotationBeanPostProcessor.java:585) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
… 18 common frames omitted
Caused by: java.lang.IllegalStateException: Duplicate key org.esa.sen2agri.entities.DataSourceConfiguration@4b7f29e
at java.util.stream.Collectors.lambda$throwingMerger$0(Collectors.java:133) ~[na:1.8.0_92]
at java.util.HashMap.merge(HashMap.java:1253) ~[na:1.8.0_92]
at java.util.stream.Collectors.lambda$toMap$58(Collectors.java:1320) ~[na:1.8.0_92]
at java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169) ~[na:1.8.0_92]
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[na:1.8.0_92]
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374) ~[na:1.8.0_92]
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[na:1.8.0_92]
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[na:1.8.0_92]
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708) ~[na:1.8.0_92]
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[na:1.8.0_92]
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499) ~[na:1.8.0_92]
at org.esa.sen2agri.commons.Config.getDownloadConfigurations(Config.java:276) ~[services-core-1.9-SNAPSHOT.jar:na]
at org.esa.sen2agri.services.ScheduleManager.schedule(ScheduleManager.java:99) ~[services-core-1.9-SNAPSHOT.jar:na]
at org.esa.sen2agri.services.ScheduleManager.initialize(ScheduleManager.java:76) ~[services-core-1.9-SNAPSHOT.jar:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_92]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_92]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_92]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_92]
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:366) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:311) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:134) ~[spring-beans-4.3.11.RELEASE.jar:4.3.11.RELEASE]
… 30 common frames omitted
##database:
sen2agri=# select * from datasource;
id | satellite_id | name | scope | username | passwrd | fetch_mode | max_retries | retry_interval_minutes | download_path |
specific_params | max_connections | local_root | enabled
----±-------------±--------------------±------±------------±----------±-----------±------------±-----------------------±--------------------------------±------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------±----------------±-----------±--------
21 | 1 | Scientific Data Hub | 3 | redoop | RedOop123 | 1 | 72 | 60 | /mnt/archive/dwn_def/s2/default | {“parameters”:{“dsParameter”:[{“name”:“cloudcoverpercentage”,“type”:“java.lang
.Double”,“value”:“100.0”},{“name”:“platformName”,“type”:“java.lang.String”,“value”:“Sentinel-2”},{“name”:“productType”,“type”:“java.lang.String”,“value”:“S2MSI1C”}]}} | 1 | | t
19 | 2 | Amazon Web Services | 2 | Asaf | ahmet2014 | 1 | 72 | 60 | /mnt/archive/dwn_def/l8/default | {“parameters”:{“dsParameter”:[{“name”:“cloudcoverpercentage”,“type”:“java.lang
.Double”,“value”:“100.0”},{“name”:“collection”,“type”:“java.lang.String”,“value”:“COLLECTION_1”},{“name”:“platformName”,“type”:“java.lang.String”,“value”:“Landsat-8”}]}} | 1 | | f
18 | 2 | USGS | 3 | tongxiaojun | RedOop123 | 1 | 72 | 60 | /mnt/archive/dwn_def/l8/default | {“parameters”:{“dsParameter”:[{“name”:“cloud_from”,“type”:“java.lang.Double”,"
value":“0.0”},{“name”:“cloud_to”,“type”:“java.lang.Double”,“value”:“100.0”},{“name”:“satellite_name”,“type”:“java.lang.String”,“value”:“landsat-8”}]}} | 1 | | t
20 | 3 | Scientific Data Hub | 3 | redoop | RedOop123 | 1 | 72 | 60 | /mnt/archive | {“parameters”:{“dsParameter”:[{“name”:“platformName”,“type”:“java.lang.String”
,“value”:“Sentinel-1”},{“name”:“productType”,“type”:“java.lang.String”,“value”:“SLC”}]}} | 1 | | t
22 | 1 | Amazon Web Services | 2 | Asaf | ahmet2014 | 1 | 72 | 60 | /mnt/archive/dwn_def/s2/default | {“parameters”:{“dsParameter”:[{“name”:“cloudcoverpercentage”,“type”:“java.lang
.Double”,“value”:“100.0”},{“name”:“platformName”,“type”:“java.lang.String”,“value”:“Sentinel-2”}]}} | 1 | | t
(5 rows)
when SciHubDataSource.Sentinel1.scope=1 and SciHubDataSource.Sentinel2.scope=1
2018-08-26 18:26:58.658 ERROR 21038 — [icesSentinel2-1] DownloadStrategy : Cannot download http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/products/2018/1/19/S2A_MSIL1C_20180119T101331_N0206_R022_T33UWR_20180119T135441/metadata.xml: Server returned HTTP response code: 403 for URL: http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/products/2018/1/19/S2A_MSIL1C_20180119T101331_N0206_R022_T33UWR_20180119T135441/metadata.xml
/mnt/archive/dwn_def/s2/default/slovec/S2A_MSIL1C_20180119T101331_N0206_R022_T33UWR_20180119T135441.SAFE
2018-08-26 18:26:58.659 WARN 21038 — [icesSentinel2-1] Sentinel2Strategy : Either the product S2A_MSIL1C_20180119T101331_N0206_R022_T33UWR_20180119T135441 was not found in the data bucket or the metadata file could not be downloaded
2018-08-26 18:26:58.659 WARN 21038 — [icesSentinel2-1] DownloadStrategy : Product download aborted
2018-08-26 18:26:58.669 INFO 21038 — [icesSentinel2-1] org.esa.sen2agri.scheduling.LookupJob : Product Download Completed - remaining products to complete for site ‘Slovec’ and satellite ‘S2’ = ‘86’
java.io.IOException: Server returned HTTP response code: 403 for URL: http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/products/2018/1/21/S2B_MSIL1C_20180121T100319_N0206_R122_T33UWR_20180121T121018/metadata.xml
at sun.reflect.GeneratedConstructorAccessor58.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at sun.net.www.protocol.http.HttpURLConnection$10.run(HttpURLConnection.java:1890)
at sun.net.www.protocol.http.HttpURLConnection$10.run(HttpURLConnection.java:1885)
at java.security.AccessController.doPrivileged(Native Method)
at sun.net.www.protocol.http.HttpURLConnection.getChainedException(HttpURLConnection.java:1884)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1457)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1441)
at ro.cs.tao.datasource.remote.DownloadStrategy.downloadFile(DownloadStrategy.java:463)
at ro.cs.tao.datasource.remote.DownloadStrategy.downloadFile(DownloadStrategy.java:290)
at ro.cs.tao.datasource.remote.DownloadStrategy.downloadFile(DownloadStrategy.java:286)
at ro.cs.tao.datasource.remote.aws.download.Sentinel2Strategy.fetchImpl(Sentinel2Strategy.java:151)
at ro.cs.tao.datasource.remote.DownloadStrategy.fetch(DownloadStrategy.java:212)
at ro.cs.tao.datasource.DataSourceComponent.doFetch(DataSourceComponent.java:303)
at org.esa.sen2agri.services.internal.DownloadServiceImpl.download(DownloadServiceImpl.java:225)
at org.esa.sen2agri.scheduling.LookupJob.lambda$lookupAndDownload$3(LookupJob.java:206)
at org.esa.sen2agri.scheduling.LookupJob$DownloadTask.run(LookupJob.java:298)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: Server returned HTTP response code: 403 for URL: http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/products/2018/1/21/S2B_MSIL1C_20180121T100319_N0206_R122_T33UWR_20180121T121018/metadata.xml
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1840)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1441)
at sun.net.www.protocol.http.HttpURLConnection.getHeaderField(HttpURLConnection.java:2943)
at java.net.URLConnection.getHeaderFieldLong(URLConnection.java:629)
at java.net.URLConnection.getContentLengthLong(URLConnection.java:501)
at ro.cs.tao.datasource.remote.DownloadStrategy.downloadFile(DownloadStrategy.java:431)
… 13 more
Thanks