SlaveTemplate.java 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442
  1. package hudson.plugins.ec2;
  2. import hudson.Extension;
  3. import hudson.Util;
  4. import hudson.model.Describable;
  5. import hudson.model.TaskListener;
  6. import hudson.model.Descriptor;
  7. import hudson.model.Descriptor.FormException;
  8. import hudson.model.Hudson;
  9. import hudson.model.Label;
  10. import hudson.model.Node;
  11. import hudson.model.labels.LabelAtom;
  12. import hudson.util.FormValidation;
  13. import hudson.util.ListBoxModel;
  14. import java.io.IOException;
  15. import java.io.PrintStream;
  16. import java.util.*;
  17. import javax.servlet.ServletException;
  18. import org.apache.commons.codec.binary.Base64;
  19. import org.apache.commons.lang.StringUtils;
  20. import org.kohsuke.stapler.DataBoundConstructor;
  21. import org.kohsuke.stapler.QueryParameter;
  22. import com.amazonaws.AmazonClientException;
  23. import com.amazonaws.services.ec2.AmazonEC2;
  24. import com.amazonaws.services.ec2.model.*;
  25. /**
  26. * Template of {@link EC2Slave} to launch.
  27. *
  28. * @author Kohsuke Kawaguchi
  29. */
  30. public class SlaveTemplate implements Describable<SlaveTemplate> {
  31. public final String ami;
  32. public final String description;
  33. public final String zone;
  34. public final String securityGroups;
  35. public final String remoteFS;
  36. public final String sshPort;
  37. public final InstanceType type;
  38. public final String labels;
  39. public final String initScript;
  40. public final String userData;
  41. public final String numExecutors;
  42. public final String remoteAdmin;
  43. public final String rootCommandPrefix;
  44. public final String jvmopts;
  45. public final String subnetId;
  46. public final String idleTerminationMinutes;
  47. public final int instanceCap;
  48. public final boolean stopOnTerminate;
  49. private final List<EC2Tag> tags;
  50. public final boolean usePrivateDnsName;
  51. protected transient EC2Cloud parent;
  52. private transient /*almost final*/ Set<LabelAtom> labelSet;
  53. private transient /*almost final*/ Set<String> securityGroupSet;
  54. @DataBoundConstructor
  55. public SlaveTemplate(String ami, String zone, String securityGroups, String remoteFS, String sshPort, InstanceType type, String labelString, String description, String initScript, String userData, String numExecutors, String remoteAdmin, String rootCommandPrefix, String jvmopts, boolean stopOnTerminate, String subnetId, List<EC2Tag> tags, String idleTerminationMinutes, boolean usePrivateDnsName, String instanceCapStr) {
  56. this.ami = ami;
  57. this.zone = zone;
  58. this.securityGroups = securityGroups;
  59. this.remoteFS = remoteFS;
  60. this.sshPort = sshPort;
  61. this.type = type;
  62. this.labels = Util.fixNull(labelString);
  63. this.description = description;
  64. this.initScript = initScript;
  65. this.userData = userData;
  66. this.numExecutors = Util.fixNull(numExecutors).trim();
  67. this.remoteAdmin = remoteAdmin;
  68. this.rootCommandPrefix = rootCommandPrefix;
  69. this.jvmopts = jvmopts;
  70. this.stopOnTerminate = stopOnTerminate;
  71. this.subnetId = subnetId;
  72. this.tags = tags;
  73. this.idleTerminationMinutes = idleTerminationMinutes;
  74. this.usePrivateDnsName = usePrivateDnsName;
  75. if (null == instanceCapStr || instanceCapStr.equals("")) {
  76. this.instanceCap = Integer.MAX_VALUE;
  77. } else {
  78. this.instanceCap = Integer.parseInt(instanceCapStr);
  79. }
  80. readResolve(); // initialize
  81. }
  82. public EC2Cloud getParent() {
  83. return parent;
  84. }
  85. public String getLabelString() {
  86. return labels;
  87. }
  88. public String getDisplayName() {
  89. return description+" ("+ami+")";
  90. }
  91. String getZone() {
  92. return zone;
  93. }
  94. public String getSecurityGroupString() {
  95. return securityGroups;
  96. }
  97. public Set<String> getSecurityGroupSet() {
  98. return securityGroupSet;
  99. }
  100. public Set<String> parseSecurityGroups() {
  101. if (securityGroups == null || "".equals(securityGroups.trim())) {
  102. return Collections.emptySet();
  103. } else {
  104. return new HashSet<String>(Arrays.asList(securityGroups.split("\\s*,\\s*")));
  105. }
  106. }
  107. public int getNumExecutors() {
  108. try {
  109. return Integer.parseInt(numExecutors);
  110. } catch (NumberFormatException e) {
  111. return EC2Slave.toNumExecutors(type);
  112. }
  113. }
  114. public int getSshPort() {
  115. try {
  116. return Integer.parseInt(sshPort);
  117. } catch (NumberFormatException e) {
  118. return 22;
  119. }
  120. }
  121. public String getRemoteAdmin() {
  122. return remoteAdmin;
  123. }
  124. public String getRootCommandPrefix() {
  125. return rootCommandPrefix;
  126. }
  127. public String getSubnetId() {
  128. return subnetId;
  129. }
  130. public List<EC2Tag> getTags() {
  131. if (null == tags) return null;
  132. return Collections.unmodifiableList(tags);
  133. }
  134. public String getidleTerminationMinutes() {
  135. return idleTerminationMinutes;
  136. }
  137. public Set<LabelAtom> getLabelSet(){
  138. return labelSet;
  139. }
  140. public int getInstanceCap() {
  141. return instanceCap;
  142. }
  143. public String getInstanceCapStr() {
  144. if (instanceCap==Integer.MAX_VALUE) {
  145. return "";
  146. } else {
  147. return String.valueOf(instanceCap);
  148. }
  149. }
  150. /**
  151. * Does this contain the given label?
  152. *
  153. * @param l
  154. * can be null to indicate "don't care".
  155. */
  156. public boolean containsLabel(Label l) {
  157. return l==null || labelSet.contains(l);
  158. }
  159. /**
  160. * Provisions a new EC2 slave.
  161. *
  162. * @return always non-null. This needs to be then added to {@link Hudson#addNode(Node)}.
  163. */
  164. public EC2Slave provision(TaskListener listener) throws AmazonClientException, IOException {
  165. PrintStream logger = listener.getLogger();
  166. AmazonEC2 ec2 = getParent().connect();
  167. try {
  168. logger.println("Launching "+ami);
  169. KeyPair keyPair = parent.getPrivateKey().find(ec2);
  170. if(keyPair==null) {
  171. throw new AmazonClientException("No matching keypair found on EC2. Is the EC2 private key a valid one?");
  172. }
  173. RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
  174. List<Filter> diFilters = new ArrayList<Filter>();
  175. diFilters.add(new Filter("image-id").withValues(ami));
  176. if (StringUtils.isNotBlank(getZone())) {
  177. Placement placement = new Placement(getZone());
  178. riRequest.setPlacement(placement);
  179. diFilters.add(new Filter("availability-zone").withValues(getZone()));
  180. }
  181. if (StringUtils.isNotBlank(getSubnetId())) {
  182. riRequest.setSubnetId(getSubnetId());
  183. diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));
  184. /* If we have a subnet ID then we can only use VPC security groups */
  185. if (!securityGroupSet.isEmpty()) {
  186. List<String> group_ids = new ArrayList<String>();
  187. DescribeSecurityGroupsRequest group_req = new DescribeSecurityGroupsRequest();
  188. group_req.withFilters(new Filter("group-name").withValues(securityGroupSet));
  189. DescribeSecurityGroupsResult group_result = ec2.describeSecurityGroups(group_req);
  190. for (SecurityGroup group : group_result.getSecurityGroups()) {
  191. if (group.getVpcId() != null && !group.getVpcId().isEmpty()) {
  192. List<Filter> filters = new ArrayList<Filter>();
  193. filters.add(new Filter("vpc-id").withValues(group.getVpcId()));
  194. filters.add(new Filter("state").withValues("available"));
  195. filters.add(new Filter("subnet-id").withValues(getSubnetId()));
  196. DescribeSubnetsRequest subnet_req = new DescribeSubnetsRequest();
  197. subnet_req.withFilters(filters);
  198. DescribeSubnetsResult subnet_result = ec2.describeSubnets(subnet_req);
  199. List subnets = subnet_result.getSubnets();
  200. if(subnets != null && !subnets.isEmpty()) {
  201. group_ids.add(group.getGroupId());
  202. }
  203. }
  204. }
  205. if (securityGroupSet.size() != group_ids.size()) {
  206. throw new AmazonClientException( "Security groups must all be VPC security groups to work in a VPC context" );
  207. }
  208. if (!group_ids.isEmpty()) {
  209. riRequest.setSecurityGroupIds(group_ids);
  210. diFilters.add(new Filter("instance.group-id").withValues(group_ids));
  211. }
  212. }
  213. } else {
  214. /* No subnet: we can use standard security groups by name */
  215. riRequest.setSecurityGroups(securityGroupSet);
  216. if (securityGroupSet.size() > 0)
  217. diFilters.add(new Filter("group-name").withValues(securityGroupSet));
  218. }
  219. String userDataString = Base64.encodeBase64String(userData.getBytes());
  220. riRequest.setUserData(userDataString);
  221. riRequest.setKeyName(keyPair.getKeyName());
  222. diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
  223. riRequest.setInstanceType(type.toString());
  224. diFilters.add(new Filter("instance-type").withValues(type.toString()));
  225. HashSet<Tag> inst_tags = null;
  226. if (tags != null && !tags.isEmpty()) {
  227. inst_tags = new HashSet<Tag>();
  228. for(EC2Tag t : tags) {
  229. diFilters.add(new Filter("tag:"+t.getName()).withValues(t.getValue()));
  230. }
  231. }
  232. DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
  233. diFilters.add(new Filter("instance-state-name").withValues(InstanceStateName.Stopped.toString(),
  234. InstanceStateName.Stopping.toString()));
  235. diRequest.setFilters(diFilters);
  236. logger.println("Looking for existing instances: "+diRequest);
  237. DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
  238. if (diResult.getReservations().size() == 0) {
  239. // Have to create a new instance
  240. Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);
  241. /* Now that we have our instance, we can set tags on it */
  242. if (inst_tags != null) {
  243. CreateTagsRequest tag_request = new CreateTagsRequest();
  244. tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
  245. ec2.createTags(tag_request);
  246. // That was a remote request - we should also update our local instance data.
  247. inst.setTags(inst_tags);
  248. }
  249. logger.println("No existing instance found - created: "+inst);
  250. return newSlave(inst);
  251. }
  252. Instance inst = diResult.getReservations().get(0).getInstances().get(0);
  253. logger.println("Found existing stopped instance: "+inst);
  254. List<String> instances = new ArrayList<String>();
  255. instances.add(inst.getInstanceId());
  256. StartInstancesRequest siRequest = new StartInstancesRequest(instances);
  257. StartInstancesResult siResult = ec2.startInstances(siRequest);
  258. logger.println("Starting existing instance: "+inst+ " result:"+siResult);
  259. List<Node> nodes = Hudson.getInstance().getNodes();
  260. for (int i = 0, len = nodes.size(); i < len; i++) {
  261. if (!(nodes.get(i) instanceof EC2Slave))
  262. continue;
  263. EC2Slave ec2Node = (EC2Slave) nodes.get(i);
  264. if (ec2Node.getInstanceId().equals(inst.getInstanceId())) {
  265. logger.println("Found existing corresponding: "+ec2Node);
  266. return ec2Node;
  267. }
  268. }
  269. // Existing slave not found
  270. logger.println("Creating new slave for existing instance: "+inst);
  271. return newSlave(inst);
  272. } catch (FormException e) {
  273. throw new AssertionError(); // we should have discovered all configuration issues upfront
  274. }
  275. }
  276. private EC2Slave newSlave(Instance inst) throws FormException, IOException {
  277. return new EC2Slave(inst.getInstanceId(), description, remoteFS, getSshPort(), getNumExecutors(), labels, initScript, remoteAdmin, rootCommandPrefix, jvmopts, stopOnTerminate, idleTerminationMinutes, inst.getPublicDnsName(), inst.getPrivateDnsName(), EC2Tag.fromAmazonTags(inst.getTags()), usePrivateDnsName);
  278. }
  279. /**
  280. * Provisions a new EC2 slave based on the currently running instance on EC2,
  281. * instead of starting a new one.
  282. */
  283. public EC2Slave attach(String instanceId, TaskListener listener) throws AmazonClientException, IOException {
  284. PrintStream logger = listener.getLogger();
  285. AmazonEC2 ec2 = getParent().connect();
  286. try {
  287. logger.println("Attaching to "+instanceId);
  288. DescribeInstancesRequest request = new DescribeInstancesRequest();
  289. request.setInstanceIds(Collections.singletonList(instanceId));
  290. Instance inst = ec2.describeInstances(request).getReservations().get(0).getInstances().get(0);
  291. return newSlave(inst);
  292. } catch (FormException e) {
  293. throw new AssertionError(); // we should have discovered all configuration issues upfront
  294. }
  295. }
  296. /**
  297. * Initializes data structure that we don't persist.
  298. */
  299. protected Object readResolve() {
  300. labelSet = Label.parse(labels);
  301. securityGroupSet = parseSecurityGroups();
  302. return this;
  303. }
  304. public Descriptor<SlaveTemplate> getDescriptor() {
  305. return Hudson.getInstance().getDescriptor(getClass());
  306. }
  307. @Extension
  308. public static final class DescriptorImpl extends Descriptor<SlaveTemplate> {
  309. @Override
  310. public String getDisplayName() {
  311. return null;
  312. }
  313. /**
  314. * Since this shares much of the configuration with {@link EC2Computer}, check its help page, too.
  315. */
  316. @Override
  317. public String getHelpFile(String fieldName) {
  318. String p = super.getHelpFile(fieldName);
  319. if (p==null) p = Hudson.getInstance().getDescriptor(EC2Slave.class).getHelpFile(fieldName);
  320. return p;
  321. }
  322. /***
  323. * Check that the AMI requested is available in the cloud and can be used.
  324. */
  325. public FormValidation doValidateAmi(
  326. @QueryParameter String accessId, @QueryParameter String secretKey,
  327. @QueryParameter String region,
  328. final @QueryParameter String ami) throws IOException, ServletException {
  329. AmazonEC2 ec2 = EC2Cloud.connect(accessId, secretKey, AmazonEC2Cloud.getEc2EndpointUrl(region));
  330. if(ec2!=null) {
  331. try {
  332. List<String> images = new LinkedList<String>();
  333. images.add(ami);
  334. List<String> owners = new LinkedList<String>();
  335. List<String> users = new LinkedList<String>();
  336. DescribeImagesRequest request = new DescribeImagesRequest();
  337. request.setImageIds(images);
  338. request.setOwners(owners);
  339. request.setExecutableUsers(users);
  340. List<Image> img = ec2.describeImages(request).getImages();
  341. if(img==null || img.isEmpty())
  342. // de-registered AMI causes an empty list to be returned. so be defensive
  343. // against other possibilities
  344. return FormValidation.error("No such AMI, or not usable with this accessId: "+ami);
  345. return FormValidation.ok(img.get(0).getImageLocation()+" by "+img.get(0).getImageOwnerAlias());
  346. } catch (AmazonClientException e) {
  347. return FormValidation.error(e.getMessage());
  348. }
  349. } else
  350. return FormValidation.ok(); // can't test
  351. }
  352. public FormValidation doCheckIdleTerminationMinutes(@QueryParameter String value) {
  353. if (value == null || value.trim() == "") return FormValidation.ok();
  354. try {
  355. int val = Integer.parseInt(value);
  356. if (val >= 0) return FormValidation.ok();
  357. }
  358. catch ( NumberFormatException nfe ) {}
  359. return FormValidation.error("Idle Termination time must be a non-negative integer (or null)");
  360. }
  361. public FormValidation doCheckInstanceCapStr(@QueryParameter String value) {
  362. if (value == null || value.trim() == "") return FormValidation.ok();
  363. try {
  364. int val = Integer.parseInt(value);
  365. if (val >= 0) return FormValidation.ok();
  366. } catch ( NumberFormatException nfe ) {}
  367. return FormValidation.error("InstanceCap must be a non-negative integer (or null)");
  368. }
  369. public ListBoxModel doFillZoneItems( @QueryParameter String accessId,
  370. @QueryParameter String secretKey,
  371. @QueryParameter String region)
  372. throws IOException, ServletException
  373. {
  374. return EC2Slave.fillZoneItems(accessId, secretKey, region);
  375. }
  376. }
  377. }